yum – dump all yum repos configuration directives

Per the man page, the yum-config-manager is “a program that can manage main yum configuration options, toggle which repositories are enabled or disabled, and add new repositories.” The details on how to use the command is in the Official Redhat documentation.

One feature that the man page does not list is how you can use the yum-config-manager to display the yum repo configuration sections/directives and options. Not only can you use it to just show the configuration in your system, but it can also help you with displaying all the options supported by yum configuration. It might be useful for scripting as well.

Installation – identify the package name:

yum whatprovides */yum-config-manager

Install package –

yum install yum-utils

Once the package is installed, the command yum-config-manager should be available –

[root@kauai /tmp]# which yum-config-manager
/usr/bin/yum-config-manager

Running yum-config-manager will dump a list of all repositories in the server, and for each repository it will list all directives, including the hidden ones.

Below is just a the truncated version of the output, the output is much more longer depending on the number of yum repositories in your system –

[root@kauai /tmp]# yum-config-manager
===================================== main =====================================
[main]
alwaysprompt = True
assumeno = False
assumeyes = False
bandwidth = 0
bugtracker_url = http://bugs.centos.org/set_project.php?project_id=19&ref=http://bugs.centos.org/bug_report_page.php?category=yum
cache = 0
cachedir = /var/cache/yum/x86_64/6
clean_requirements_on_remove = False
color = auto
color_list_available_downgrade = dim,cyan
color_list_available_install = normal
color_list_available_reinstall = bold,underline,green
color_list_available_upgrade = bold,blue
color_list_installed_extra = bold,red
color_list_installed_newer = bold,yellow
color_list_installed_older = bold
color_list_installed_reinstall = normal
color_search_match = bold
color_update_installed = normal
color_update_local = bold
color_update_remote = normal
commands = 
debuglevel = 2
depsolve_loop_limit = 100
diskspacecheck = True
distroverpkg = centos-release
downloaddir = 
downloadonly = 
enable_group_conditionals = True
enabled = True
enablegroups = True
errorlevel = 2
exactarch = True
exactarchlist = 
exclude = 
exit_on_lock = False
failovermethod = priority
ftp_disable_epsv = False
gaftonmode = False
gpgcheck = True
group_package_types = mandatory,
   default
groupremove_leaf_only = False
history_list_view = users
history_record = True
history_record_packages = yum,
   rpm
http_caching = all
installonly_limit = 5
installonlypkgs = kernel,
   kernel-bigmem,
   installonlypkg(kernel-module),
   installonlypkg(vm),
   kernel-enterprise,
   kernel-smp,
   kernel-debug,
   kernel-unsupported,
   kernel-source,
   kernel-devel,
   kernel-PAE,
   kernel-PAE-debug
installroot = /
keepalive = True
keepcache = False
kernelpkgnames = kernel,
   kernel-smp,
   kernel-enterprise,
   kernel-bigmem,
   kernel-BOOT,
   kernel-PAE,
   kernel-PAE-debug
loadts_ignoremissing = False
loadts_ignorerpm = False
localpkg_gpgcheck = False
logfile = /var/log/yum.log
mdpolicy = group:primary
metadata_expire = 21600
mirrorlist_expire = 86400
multilib_policy = best
obsoletes = True
overwrite_groups = False
password = 
persistdir = /var/lib/yum
pluginconfpath = /etc/yum/pluginconf.d
pluginpath = /usr/share/yum-plugins,
   /usr/lib/yum-plugins
plugins = True
progess_obj = 
protected_multilib = True
protected_packages = yum
proxy = False
proxy_password = 
proxy_username = 
query_install_excludes = False
recent = 7
recheck_installed_requires = True
repo_gpgcheck = False
reposdir = /etc/yum/repos.d,
   /etc/yum.repos.d
reset_nice = True
retries = 10
rpm_check_debug = True
rpmverbosity = info
showdupesfromrepos = False
skip_broken = False
ssl_check_cert_permissions = True
sslcacert = 
sslclientcert = 
sslclientkey = 
sslverify = True
syslog_device = /dev/log
syslog_facility = LOG_USER
syslog_ident = 
throttle = 0
timeout = 30.0
tolerant = True
tsflags = 
username = 
================================== repo: base ==================================
[base]
bandwidth = 0
base_persistdir = /var/lib/yum/repos/x86_64/6
baseurl = 
cache = 0
cachedir = /var/cache/yum/x86_64/6/base
cost = 1000
enabled = True
enablegroups = True
exclude = 
failovermethod = priority
ftp_disable_epsv = False
gpgcadir = /var/lib/yum/repos/x86_64/6/base/gpgcadir
gpgcakey = 
gpgcheck = True
gpgdir = /var/lib/yum/repos/x86_64/6/base/gpgdir
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
hdrdir = /var/cache/yum/x86_64/6/base/headers
http_caching = all
includepkgs = 
keepalive = True
mdpolicy = group:primary
mediaid = 
metadata_expire = 21600
metalink = 
mirrorlist = http://mirrorlist.centos.org/?release=6&arch=x86_64&repo=os&infra=stock
mirrorlist_expire = 86400
name = CentOS-6 - Base
old_base_cache_dir = 
password = 
persistdir = /var/lib/yum/repos/x86_64/6/base
pkgdir = /var/cache/yum/x86_64/6/base/packages
proxy = False
proxy_dict = 
proxy_password = 
proxy_username = 
repo_gpgcheck = False
retries = 10
skip_if_unavailable = False
ssl_check_cert_permissions = True
sslcacert = 
sslclientcert = 
sslclientkey = 
sslverify = True
throttle = 0
timeout = 30.0
username = 

References –

Redhat official documentation

In part 1 of this series, we saw how to pull Docker images from Docker Hub and launch Docker containers. We interacted with a running Docker container by running some bash commands, in this tutorial we will see how to use Dockerfile to automate image building for quicker deployment of applications in a container.

Dockerfile is a text file containing a set of instructions or commands in order to build a Docker image.

Prerequisites

1. Complete the tutorial on part 1 before proceeding. You will need a Docker engine running and the latest official Ubuntu Docker images locally hosted.

2. Create directories

$mkdir ~/docker-flask 
$cd ~/docker-flask

3. Add Dockerfile : ~/docker-flask/Dockerfile
The commands below will be used to create the Docker image. It will pull the latest Ubuntu official Docker image as a first layer or base. Then it will resynchronize the apt package index files from their sources.

A /flask directory will be created in the image, followed by installing Flask and running our Flask app, which we will write in next step.

FROM ubuntu:latest
RUN apt-get update && apt-get install -y python-pip python-dev
COPY . /flask
WORKDIR /flask
RUN pip install -r requirements.txt
EXPOSE 80
ENTRYPOINT ["python"]
CMD ["app.py"]

4. Write flask app : ~/docker-flask/app.py
Let us write a practical app, rather than just printing hello world. The flask app will return the user agent information of the visitor if the index page is visited.

We will also have a URL under /status/ followed by a valid HTTP status code. Given this HTTP status code by the visitor, the flask web server will generate the same HTTP status code header. For instance, if the user visits http://localhost/status/502, the flask server will respond with ‘502 BAD GATEWAY’ HTTP header.

Let us write it under ~/docker-flask/app.py

from flask import Flask
from flask import request, jsonify

app = Flask(__name__)

@app.route('/')
def user_agent():
    user_agent = request.headers.get('User-Agent')
    return 'Your browser is %s.' % user_agent

@app.route('/status/<int:httpcode>')
def get_status(httpcode):
    httpcode = int(httpcode)
    if httpcode < 100 or httpcode >= 600:
        return jsonify({'Status': 'Invalid HTTP status code'})
    elif httpcode >= 100 and httpcode < 500:
        return jsonify({'Status': 'UP'}) , httpcode
    else:
        return jsonify({'Status': 'DOWN'}) , httpcode

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0', port=80)

5. requirements.txt : ~/docker-flask/requirements.txt

cat ~/docker-flask/requirements.txt
Flask==0.12

By now, your directory structure should look similar to this –

daniel@lindell:~/docker-flask$ pwd
/home/daniel/docker-flask

daniel@lindell:~/docker-flask$ ls
app.py  Dockerfile  requirements.txt

Time to build the Docker image –

sudo docker build -t flaskweb:latest .

This will execute the series of commands under Dockerfile. If successful, you will end up with a Docker image named flaskweb and tagged latest –

root@lindell:~# docker images
REPOSITORY          TAG                 IMAGE ID            CREATED             SIZE
flaskweb            latest              6b45443b6380        22 minutes ago      440.4 MB
ubuntu              latest              104bec311bcd        2 weeks ago         129 MB

If you encounter any errors, validate you don’t have any syntax errors on Dockerfile.

It is time to run the container –

daniel@lindell:~/docker-flask$ sudo docker run -d -p 80:80 flaskweb
d9af9a1c92bff45b56fc97d13935972b65e3554bfe22ec2f3c102fd26bd20e4c

daniel@lindell:~/docker-flask$ sudo docker ps
CONTAINER ID        IMAGE               COMMAND             CREATED              STATUS              PORTS                NAMES
d9af9a1c92bf        flaskweb            "python app.py"     About a minute ago   Up About a minute   0.0.0.0:80->80/tcp   drunk_mcnulty

In this case, both the host and container will be listening on port 80, feel free to modify this according to your setup.

Test it, we will use httpie to query the web server, if you don’t have httpie installed, you can use ‘curl -I’ to get the full header –

daniel@lindell:~/blog/docker-flask$ http http://localhost/
HTTP/1.0 200 OK
Content-Length: 29
Content-Type: text/html; charset=utf-8
Date: Fri, 30 Dec 2016 14:35:53 GMT
Server: Werkzeug/0.11.13 Python/2.7.12

Your browser is HTTPie/0.9.2.

daniel@lindell:~/blog/docker-flask$ http http://localhost/status/404
HTTP/1.0 404 NOT FOUND
Content-Length: 21
Content-Type: application/json
Date: Fri, 30 Dec 2016 14:35:56 GMT
Server: Werkzeug/0.11.13 Python/2.7.12

{
    "Status": "UP"
}

daniel@lindell:~/blog/docker-flask$ http http://localhost/status/502
HTTP/1.0 502 BAD GATEWAY
Content-Length: 23
Content-Type: application/json
Date: Fri, 30 Dec 2016 14:35:58 GMT
Server: Werkzeug/0.11.13 Python/2.7.12

{
    "Status": "DOWN"
}

Full clean up – if you want to start all over again or want to delete the container and images we have created, i have outlined the steps below. The first step is to stop the running container using ‘docker stop’ command, pass it the first few digits of the container ID.

Once the container is stopped, use ‘docker rm’ to delete the container. At this point, we can proceed with deleting the image as the image is not attached to any running container. Use ‘docker rmi’ to delete the image. We will keep the base Ubuntu image for future use.

daniel@lindell:/tmp$ sudo docker ps
CONTAINER ID        IMAGE               COMMAND             CREATED             STATUS              PORTS                NAMES
d9af9a1c92bf        flaskweb            "python app.py"     12 minutes ago      Up 12 minutes       0.0.0.0:80->80/tcp   drunk_mcnulty

daniel@lindell:/tmp$ sudo docker stop d9a
d9a

daniel@lindell:/tmp$ sudo docker rm d9a
d9a

daniel@lindell:/tmp$ sudo docker ps
CONTAINER ID        IMAGE               COMMAND             CREATED             STATUS              PORTS               NAMES

daniel@lindell:/tmp$ sudo docker images
REPOSITORY          TAG                 IMAGE ID            CREATED             SIZE
flaskweb            latest              6b45443b6380        39 minutes ago      440.4 MB
ubuntu              latest              104bec311bcd        2 weeks ago         129 MB

daniel@lindell:/tmp$ sudo docker rmi 6b45
Untagged: flaskweb:latest
Deleted: sha256:6b45443b63805583f41fbf60aaf5cf746b871fdcfa8fe1c6d5adfb52870e7c89
Deleted: sha256:02062a8ea251d993f54e15f9e5654e40894449430acd045476000cd9ebbdf459
Deleted: sha256:fa2439cd5bc8a53152877c1dc3b12a60ab808bcfe5078549ea5e945f462330da
Deleted: sha256:3bac38b223d80a4db6c4283fd56275fe05ceeab6a1dfd81871aa14c6cda387df
Deleted: sha256:d97357dc5d7454e3b7757f2c348323c84d1902dd806792c53d1fd0ca7813b091
Deleted: sha256:b55dd5bd3326ec4657dc389f4aae69c34a7ba222872f7b868eb8de69d7f69dab
Deleted: sha256:eab59ae84eb136339d08fbacd2905a1ee80a0c875e8e14a4d5184fac30445714
Deleted: sha256:588253a9066c49786fcd0121353e7f0f2cea05cebbc6b9cef67f0c823d23dce8
Deleted: sha256:fe9f27a1cb9165531a1f5149c16ebcd522422e4ac2610035bbbcada7fd0b7551
Deleted: sha256:18ca1bc40895f6f97cae28fa5707bde537ac27023762303f98912c11549431ae

daniel@lindell:/tmp$ sudo docker images
REPOSITORY          TAG                 IMAGE ID            CREATED             SIZE
ubuntu              latest              104bec311bcd        2 weeks ago         129 MB

References –
https://docs.docker.com/engine/reference/builder/
https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/
http://flask.pocoo.org/docs/0.12/quickstart/

The scheduling format for the Linux scheduling daemon cron are not easy to remember, especially if you don’t work with cron that frequently. The first reaction for most Linux sys admins when they can’t remember the ordering of fields is to type ‘man crontab’, and unfortunately this man page section does not contain the schedule format information. If you are like me, you will immediately start Googling it.

What is the best way to locate the man page for crontab scheduling format then? For one thing, you can search the man page for the key work ‘crontab’ using the command below –

daniel@linubuvma:/tmp$ man -k crontab
anacrontab (5)       - configuration file for anacron
crontab (1)          - maintain crontab files for individual users (Vixie Cron)
crontab (5)          - tables for driving cron

You see, there are two sections for crontab – section 1 describes the command usage and section 5 shows the tables we are looking for. If you are familiar with how man page section numbers are assigned, you would have immediately jumped to section 5 of the man page for crontab –


1. General commands
2. System calls
3. C library functions
4. Special files (usually devices, those found in /dev) and drivers
5. File formats and conventions
6. Games and screensavers
7. Miscellanea
8. System administration commands and daemons


Short answer to how do i see the crontab schedule format is – run

 man 5 crontab 

Per the man page, the time and date fields in order are –

field allowed values
----- --------------
minute 0-59
hour 0-23
day of month 1-31
month 1-12 (or names, see below)
day of week 0-7 (0 or 7 is Sun, or use names)


One of the most commonly used Linux system administration tools is chown, which is part of the coreutils package. It is used to change the user and/or group ownership of a given file or directory. Something to be aware of this tool is, it doesn’t change the ownership of symbolic links, as shown below –

root@linubuvma:/tmp# touch test
root@linubuvma:/tmp# ls -l test
-rw-r--r-- 1 root root 12 Dec 20 08:01 test
root@linubuvma:/tmp# ln -s test sltest
root@linubuvma:/tmp# ls -l sltest
lrwxrwxrwx 1 root root 4 Dec 20 08:01 sltest -> test
root@linubuvma:/tmp# chown daniel:daniel sltest
root@linubuvma:/tmp# ls -l sltest
lrwxrwxrwx 1 root root 4 Dec 20 08:01 sltest -> test

The reason this doesn’t work is in the man page for chown – symbolic links named by arguments are silently left unchanged unless -h is used.” By simply running chown on symbolic link without ‘-h’ option, you are changing the ownership of the target. The ‘-h’ option affects symbolic links instead of any referenced file.

root@linubuvma:/tmp# chown -h daniel:daniel sltest

root@linubuvma:/tmp# ls -l sltest
lrwxrwxrwx 1 daniel daniel 4 Dec 20 08:01 sltest -> test

Though not portable, in some distros

 chown -R 

will recursively change the owernship of all files, including symbolic link files and directories. In my case, ‘chown -R /path/to/file’ works for GNU chown which is part of the ‘GNU coreutils 8.21’ package on Ubuntu 14.04.

How to record your ssh session using screen.

Per the man page – “Screen is a full-screen window manager that multiplexes a physical terminal between several processes (typically interactive shells)”.
Screen is most commonly used to create multiple sessions to remote hosts within a single terminal window or even run multiple commands locally without leaving your shell terminal. For instance, you could be tailing the log file in one session, then run a long process, then ssh into other machine etc. all within a single window.

Screen is the go to tool when setting up a remote connection, such as ssh, and you want to continue your work at any time or from any other host without worrying of a dropped connection.

In this post, I will show you how you can record your bash session.

Installation –

yum install screen        (Debian/Ubuntu)
apt-get install screen    (Redhat/CentOS)

My local environment and the remote host I am sshing to –

daniel@linubuvma:/tmp$ screen -v
Screen version 4.01.00devel (GNU) 2-May-06
daniel@linubuvma:/tmp$ uname -r
3.13.0-106-generic
daniel@linubuvma:/tmp$ cat /etc/issue
Ubuntu 14.04.5 LTS \n \l

daniel@linubuvma:/tmp$ ssh ns2 'uname -r ; cat /etc/issue'
2.6.32-642.6.1.el6.x86_64
CentOS release 6.8 (Final)
Kernel \r on an \m

The ‘-L’ option of screen is used to record your session, the session log is automatically saved in a file named ‘screenlog.n’ in your current directory.

daniel@linubuvma:/tmp$ ls
config-err-hbzs5e  one          ssh-4yheApHRgMBF  ssh-RK7GpeFuzUB8  VMwareDnD    vmware-root-2347660412
gpg-kZux7q         screenlog.0  ssh-BBblvGtb5284  vmware-daniel     vmware-root
daniel@linubuvma:/tmp$ free -m
             total       used       free     shared    buffers     cached
Mem:          3946       2489       1457          6        547       1031
-/+ buffers/cache:        911       3035
Swap:         4092          0       4092
daniel@linubuvma:/tmp$ exit
[screen is terminating]
daniel@linubuvma:/tmp$ 

The whole bash session will be logged in screenlog.0 in this case –

daniel@linubuvma:/tmp$ cat screenlog.0 
daniel@linubuvma:/tmp$ ls
config-err-hbzs5e  one          ssh-4yheApHRgMBF  ssh-RK7GpeFuzUB8  VMwareDnD    vmware-root-2347660412
gpg-kZux7q         screenlog.0  ssh-BBblvGtb5284  vmware-daniel     vmware-root
daniel@linubuvma:/tmp$ free -m
             total       used       free     shared    buffers     cached
Mem:          3946       2489       1457          6        547       1031
-/+ buffers/cache:        911       3035
Swap:         4092          0       4092
daniel@linubuvma:/tmp$ exit
exit
daniel@linubuvma:/tmp$ 

Recording your session of an ssh connection to a remote host is also similar, with ‘-L’ option followed by the command to ssh to remote host.
Option -fn (with no flow-control)
Option -t (title bar name) in this case ‘practice’.

daniel@linubuvma:/tmp$ screen -fn -t practice -L  ssh ns2
Last login: Tue Dec 27 09:46:10 2016 from linubuvma.home.net

[daniel@kauai ~]$ hostname -f
kauai.example.net
[daniel@kauai ~]$ uptime
 10:08:18 up 18 days, 10:02, 14 users,  load average: 0.19, 0.49, 0.64
[daniel@kauai ~]$ exit
[screen is terminating]


daniel@linubuvma:/tmp$ cat screenlog.0 
Last login: Tue Dec 27 09:46:10 2016 from linubuvma.home.net
[daniel@kauai ~]$ hostname -f
kauai.example.net
[daniel@kauai ~]$ uptime
 10:08:18 up 18 days, 10:02, 14 users,  load average: 0.19, 0.49, 0.64
[daniel@kauai ~]$ exit
logout
Connection to ns2 closed.
daniel@linubuvma:/tmp$ 

Additional resources –

https://www.rackaid.com/blog/linux-screen-tutorial-and-how-to/
https://linux.die.net/man/1/screen

Randomly ordering files in a directory with python

I have a playlist file which contains audio files to play. The audio player unfortunately plays the music files in a sequential order, in whatever order they are listed in the playlist file. So occasionally I have to regenerate the playlist file to randomize the audio files order. Here is a simple script that I had to write for this purpose, the core component is the random.shuffle(list) python function –

Create script file as shuffle_files.py –

#!/usr/bin/env python

import os
import random
import sys

music_files=[]

if len(sys.argv) != 2:
  print "Usage:", sys.argv[0], "/path/directory"
else:
  dir_name=sys.argv[1]
  if os.path.isdir(dir_name):
    for file_name in os.listdir(dir_name):
      music_files.append(file_name)
  else:
    print "Directory", dir_name, "does not exist"
    sys.exit(1)
# shuffle list
random.shuffle(music_files)
for item in music_files:
  print os.path.join(dir_name,item)

Run the script by providing a path to a directory with files. Each iteration should list the files in the directory in a different order.
Note – the script does not recurse into the directories, it can be easily modified with os.walk if necessary.

root@svm1010:/home/daniel/scripts# python shuffle_files.py /opt/iotop/iotop
/opt/iotop/iotop/setup.py
/opt/iotop/iotop/README
/opt/iotop/iotop/iotop
/opt/iotop/iotop/iotop.8
/opt/iotop/iotop/NEWS
/opt/iotop/iotop/iotop.py
/opt/iotop/iotop/PKG-INFO
/opt/iotop/iotop/THANKS
/opt/iotop/iotop/sbin
/opt/iotop/iotop/setup.cfg
/opt/iotop/iotop/ChangeLog
/opt/iotop/iotop/.gitignore
/opt/iotop/iotop/COPYING


root@svm1010:/home/daniel/scripts# python shuffle_files.py /opt/iotop/iotop
/opt/iotop/iotop/PKG-INFO
/opt/iotop/iotop/COPYING
/opt/iotop/iotop/iotop
/opt/iotop/iotop/setup.cfg
/opt/iotop/iotop/NEWS
/opt/iotop/iotop/README
/opt/iotop/iotop/.gitignore
/opt/iotop/iotop/setup.py
/opt/iotop/iotop/THANKS
/opt/iotop/iotop/iotop.py
/opt/iotop/iotop/ChangeLog
/opt/iotop/iotop/iotop.8
/opt/iotop/iotop/sbin


root@svm1010:/home/daniel/scripts# python shuffle_files.py /opt/iotop/iotop
/opt/iotop/iotop/THANKS
/opt/iotop/iotop/setup.py
/opt/iotop/iotop/NEWS
/opt/iotop/iotop/README
/opt/iotop/iotop/iotop.8
/opt/iotop/iotop/.gitignore
/opt/iotop/iotop/ChangeLog
/opt/iotop/iotop/sbin
/opt/iotop/iotop/PKG-INFO
/opt/iotop/iotop/iotop
/opt/iotop/iotop/COPYING
/opt/iotop/iotop/iotop.py
/opt/iotop/iotop/setup.cfg

Reference – https://docs.python.org/2/library/random.html?highlight=shuffle#random.shuffle