init

all.yml

@@ -0,0 +1,2 @@
+- import_playbook: hv.yml
+- import_playbook: containers.yml

ansible.cfg

@@ -0,0 +1,13 @@
+[defaults]
+interpreter_python = auto_silent
+stdout_callback = debug
+use_persistent_connections = true
+forks = 6
+internal_poll_interval = 0.01
+jinja2_native = true
+
+[ssh_connection]
+pipelining = true
+transfer_method = scp
+scp_if_ssh = smart
+ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s -o PreferredAuthentications=publickey,password

containers.yml

@@ -0,0 +1,21 @@
+- hosts: containers
+  gather_facts: no
+  serial: 1
+  strategy: linear
+  tasks:
+    - name: include common container role
+      include_role:
+        name: container
+
+
+    - name: check if host role exists
+      stat:
+        path: "{{ (playbook_dir, 'roles', host_role | d(inventory_hostname), 'tasks/main.yml') | path_join }}"
+      register: result
+      delegate_to: localhost
+
+
+    - name: include host role
+      include_role:
+        name: "{{ host_role | d(inventory_hostname) }}"
+      when: result.stat.exists

group_vars/all.yml

@@ -0,0 +1,5 @@
+container_config_dir: /opt/conf
+container_logs_dir: /opt/logs
+container_root_build_dir: /opt/build
+
+docker_remote_port: 5011

group_vars/containers.yml

@@ -0,0 +1,4 @@
+ansible_connection: docker
+hypervisor_hostname: hv
+
+container_default_config:

hosts

@@ -0,0 +1,87 @@
+all:
+  children:
+    nodes:
+      hosts:
+        hv:
+          ansible_host: 10.61.1.1
+          ansible_password: BJDGSLAbKyx3TyHddciZJ43mg5SSaJhQ
+          ansible_ssh_extra_args: -o StrictHostKeyChecking=no
+
+    containers:
+      hosts:
+        nginx:
+          ansible_host: 10.90.0.2
+          ansible_password: k3PpKHF52zgUaowB73V6ggrnMUDUcMeS
+          container_config:
+            image: nginx
+            custom_image: yes
+          nginx_config:
+            root:
+              error_log: /var/log/nginx/error.log info
+              load_module: /usr/local/nginx/modules/ngx_http_geoip2_module.so
+            http:
+              log_format: "custom escape=json '\"$time_iso8601\" \"$request_time\" \"$upstream_response_time\" \
+                           \"$remote_addr\" \"$remote_user\" \"$time_local\" \"$request\" \"$status\" \
+                           \"$body_bytes_sent\" \"$http_referer\" \"$http_user_agent\" \"$geoip_country_code\"'"
+
+        apache1:
+          ansible_host: 10.90.0.3
+          ansible_password: qUTuyFHzCFGVswGYEHv5MU2JzQGt9Tx7
+          host_role: apache
+          container_config:
+            image: php:8-apache
+            mounts: /opt/www/html:/var/www/html
+
+        apache2:
+          ansible_host: 10.90.0.4
+          ansible_password: F2a4v4LoQ5U6rwAgsrSt68SJwmHGARuP
+          host_role: apache
+          container_config:
+            image: php:8-apache
+            mounts: /opt/www/html:/var/www/html
+
+        cadvisor:
+          ansible_host: 10.90.0.5
+          ansible_password: dJpxV9gri438UbC82bYXCp4BeAGHyoZn
+          container_config:
+            image: gcr.io/cadvisor/cadvisor:latest
+            mounts:
+              - /:/rootfs:ro
+              - /var/run:/var/run:ro
+              - /sys:/sys:ro
+              - /var/lib/docker/:/var/lib/docker:ro
+              - /dev/disk/:/dev/disk:ro
+            privileged: yes
+            devices:
+              - /dev/kmsg
+
+        node_exporter:
+          ansible_host: 10.90.0.6
+          ansible_password: QnSKHZ5e82YnCzbS75PMHXZqhwpFbzcu
+          container_config:
+            image: prom/node-exporter:latest
+            extra_mounts:
+              - /:/host:ro,rslave
+            command: '--path.rootfs=/host'
+
+        nginx_exporter:
+          ansible_host: 10.90.0.7
+          ansible_password: rkeE2vHp2jmLjsyz4r9ASe63GM5t6FbA
+          container_config:
+            image: nginx/nginx-prometheus-exporter
+
+        mysql_exporter:
+          ansible_host: 10.61.1.7
+          ansible_password: VxMNsdHg6Fu5Ah9GxRTcQFVTGFRagAHD
+        prometheus:
+          ansible_host: 10.61.1.9
+          ansible_password: XPJEzq3ohbu3KxcqWnT65M9uX5Utxyx2
+        grafana:
+          ansible_host: 10.61.1.10
+          ansible_password: ZsA8if5Cm4sEr299SjoMEMgqV5kBYn4a
+        fluentd:
+          ansible_host: 10.61.1.11
+          ansible_password: mcLjXRpVGvxk7x4NPhuHVf9n6o8nk3cS
+        mysql:
+          ansible_host: 10.61.1.12
+          ansible_password: cr9GvFGoqnzAdUNRvTr2DRqwQqqKLMWV

hv.yml

@@ -0,0 +1,6 @@
+- hosts: hv
+  gather_facts: no
+  serial: 1
+  strategy: linear
+  roles:
+    - hv

roles/apache/defaults/main.yml

@@ -0,0 +1 @@
+apache_www_host_dir: /opt/www/html

roles/apache/tasks/prepare_build.yml

@@ -0,0 +1,16 @@
+- name: set container mount dirs
+  set_fact:
+    container_logs_mount: /var/log/apache2
+
+
+- name: create www dir
+  file:
+    path: "{{ apache_www_host_dir }}"
+    state: directory
+
+
+- name: template php script
+  template:
+    src: script.j2
+    dest: "{{ (apache_www_host_dir, 'index.php') | path_join }}"
+    force: yes

roles/apache/templates/script.j2

@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html lang="ru">
+
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+
+    <title>PHP script</title>
+</head>
+
+<body>
+    <?php
+        $sleepTime = mt_rand(1000, 4000);
+        usleep($sleepTime * 1000);
+
+        echo '<p>container hostname (PHP): ' . gethostname() . '</p>';
+        echo '<p>uptime: ' . shell_exec('uptime -p') . '</p>';
+        echo '<p>sleep time: ' . $sleepTime . '</p>';
+    ?>
+</body>
+
+</html>

roles/container/defaults/main.yml

@@ -0,0 +1,4 @@
+container_custom_image_prefix: custom-
+
+container_default_config:
+  custom_image: no

roles/container/tasks/main.yml

@@ -0,0 +1,95 @@
+- name: ensure hypervisor_hostname is present
+  fail:
+    msg: "hypervisor_hostname must be defined and must be a string"
+  when: hypervisor_hostname is not string
+
+
+- block:
+    - name: check connection to hypervisor
+      ping:
+
+
+    - name: unset per-role mount vars
+      set_fact:
+        container_config_mount: "{{ None }}"
+        container_logs_mount: "{{ None }}"
+        container_role_config: "{{ {} }}"
+
+
+    - name: check if role supports prebuilding
+      stat:
+        path: "{{ (role_path, '..', host_role | d(inventory_hostname), 'tasks/prepare_build.yml') | path_join }}"
+      register: result
+      delegate_to: localhost
+
+
+    - name: prepare build environment for role
+      include_role:
+        name: "{{ host_role | d(inventory_hostname) }}"
+        tasks_from: prepare_build
+      vars:
+        build_dir: "{{ container_build_dir | d(None) }}"
+        conf_dir: "{{ (container_config_dir, inventory_hostname) | path_join }}"
+        logs_dir: "{{ (container_logs_dir, inventory_hostname) | path_join }}"
+      when: result.stat.exists
+
+
+    - name: set container_cfg
+      set_fact:
+        container_cfg: "{{ container_default_config | d({}) |
+          combine(container_role_config | d({}), recursive=true) |
+          combine(container_config | d({}), recursive=true) }}"
+
+
+    - block:
+        - set_fact:
+            container_build_dir: "{{ (container_root_build_dir, container_cfg.image) | path_join }}"
+
+
+        - name: create container build dir
+          file:
+            path: "{{ container_build_dir }}"
+            state: directory
+
+
+        - name: template dockerfile to build dir
+          template:
+            src: "{{ container_cfg.image }}.Dockerfile.j2"
+            dest: "{{ (container_build_dir, 'Dockerfile') | path_join }}"
+            lstrip_blocks: no
+            force: yes
+
+
+        - name: build image from dockerfile
+          docker_image:
+            name: "{{ container_custom_image_prefix ~ container_cfg.image }}"
+            build:
+              dockerfile: "{{ (container_build_dir, 'Dockerfile') | path_join }}"
+              path: "{{ container_build_dir }}"
+            source: build
+
+      when: container_cfg.custom_image | d(false) == true
+
+
+    - name: create container
+      docker_container:
+        name: "{{ inventory_hostname }}"
+        image: "{{ (container_custom_image_prefix ~ container_cfg.image) if container_cfg.custom_image | d(false) == true
+                   else container_cfg.image }}"
+        hostname: "{{ inventory_hostname }}"
+        command_handling: correct
+        network_mode: bridge
+        networks:
+          - name: network
+            ipv4_address: "{{ ansible_host }}"
+        log_driver: local
+        detach: yes
+        restart_policy: unless-stopped
+        volumes: "{{ [((((container_config_dir, inventory_hostname) | path_join) ~ ':' ~ container_config_mount) if container_config_mount is string else None),
+                      ((((container_logs_dir, inventory_hostname) | path_join) ~ ':' ~ container_logs_mount) if container_logs_mount is string else None),
+                      (container_cfg.mounts | d(None))] | flatten(levels=1) | select() | list }}"
+        privileged: "{{ container_cfg.privileged | d(false) }}"
+        devices: "{{ omit if container_cfg.devices is not defined else ([container_cfg.devices] | flatten(levels=1)) }}"
+        command: "{{ container_cfg.command | d(omit) }}"
+
+  delegate_to: "{{ hypervisor_hostname }}"

roles/hv/defaults/main.yml

@@ -0,0 +1,8 @@
+timezone: Europe/Kirov
+gpg_keyrings_dir: /etc/apt/trusted.gpg.d
+docker_repo_url: https://download.docker.com/linux/debian
+
+
+docker_network_subnet: 10.90.0.0/16
+docker_network_gateway: 10.90.0.1
+docker_network_iprange: 10.90.0.0/16

roles/hv/handlers/main.yml

@@ -0,0 +1,11 @@
+- name: restart crond
+  service:
+    name: crond
+    state: restarted
+
+
+- name: update docker unit file
+  systemd:
+    daemon_reload: yes
+    name: docker
+    state: restarted

roles/hv/tasks/install_docker.yml

@@ -0,0 +1,153 @@
+# https://docs.docker.com/engine/install/debian/
+
+- name: ensure old docker packages are uninstalled
+  apt:
+    name:
+      - docker
+      - docker-engine
+      - docker.io
+      - containerd
+      - runc
+    force_apt_get: yes
+    purge: yes
+    state: absent
+
+
+- name: ensure apt can access https repos
+  apt:
+    name:
+      - ca-certificates
+      - curl
+      - gnupg
+      - lsb-release
+    force_apt_get: yes
+    state: latest
+
+
+- name: add keyrings dir
+  file:
+    state: directory
+    path: "{{ gpg_keyrings_dir }}"
+
+
+- name: download docker gpg key
+  get_url:
+    url: "{{ docker_repo_url ~ '/gpg' }}"
+    dest: "{{ gpg_keyrings_dir }}/docker.asc"
+    mode: a+r
+
+
+- name: add apt repo
+  apt_repository:
+    repo: "deb [arch=amd64 signed-by={{ (gpg_keyrings_dir ~ '/docker.asc') | quote }}] \
+           {{ docker_repo_url }} {{ ansible_distribution_release }} stable"
+
+
+- name: update repository index
+  apt:
+    force_apt_get: yes
+    update_cache: yes
+  changed_when: no
+
+
+- name: install docker and dependencies
+  apt:
+    name:
+      - docker-ce
+      - docker-ce-cli
+      - containerd.io
+      - docker-compose-plugin
+      - python3-pip
+    force_apt_get: yes
+    state: latest
+
+
+- name: install python docker modules
+  pip:
+    name:
+      - docker
+      - docker-compose>=1.7.0
+    state: latest
+
+
+- name: enable and start docker
+  service:
+    name: "{{ item }}"
+    enabled: yes
+    state: started
+  loop:
+    - docker
+    - containerd
+
+
+- name: flush handlers
+  meta: flush_handlers
+
+
+- name: create helloworld container
+  docker_container:
+    name: hello-world
+    image: hello-world
+    command_handling: correct
+    init: yes
+    output_logs: yes
+    log_driver: local
+    detach: no
+    network_mode: none
+  register: result
+  changed_when: no
+  failed_when: "{{ result.container.State.ExitCode != 0 or not ('Hello from Docker!' in result.container.Output) }}"
+
+
+- name: create docker network
+  docker_network:
+    name: network
+    driver: bridge
+    internal: no
+    ipam_config:
+      - subnet: "{{ docker_network_subnet }}"
+        gateway: "{{ docker_network_gateway }}"
+        iprange: "{{ docker_network_iprange }}"
+
+
+- name: save ipv4 forwarding to sysctl startup scripts
+  copy:
+    dest: /etc/sysctl.d/91-forwarding.conf
+    content: "net.ipv4.conf.all.forwarding = 1\n"
+    mode: 0644
+
+
+- name: set ipv4 forwarding
+  sysctl:
+    name: net.ipv4.conf.all.forwarding
+    value: 1
+    sysctl_set: yes
+
+
+- name: change default iptables policy
+  iptables:
+    chain: FORWARD
+    jump: ACCEPT
+
+
+- name: install iptables-persistent
+  apt:
+    name: iptables-persistent
+    force_apt_get: yes
+    state: latest
+
+
+- name: save current iptables rules
+  community.general.iptables_state:
+    ip_version: ipv4
+    table: filter
+    state: saved
+    path: /etc/iptables/rules.v4
+
+
+- name: change docker systemd service
+  lineinfile:
+    path: /lib/systemd/system/docker.service
+    regexp: '^ExecStart='
+    line: 'ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock -H tcp://0.0.0.0:{{ docker_remote_port }}'
+  notify: update docker unit file

roles/hv/tasks/main.yml

@@ -0,0 +1,46 @@
+- name: gather facts
+  setup:
+    gather_subset:
+      - min
+
+
+- name: set timezone
+  timezone:
+    name: "{{ timezone }}"
+  notify: restart crond
+
+
+- name: flush handlers to force restart of crond
+  meta: flush_handlers
+
+
+- name: enable auto reboot on kernel panic
+  copy:
+    dest: /etc/sysctl.d/90-auto-reboot.conf
+    content: "kernel.panic = 5\n"
+    mode: 0644
+
+
+- name: upgrade packages
+  include_tasks: upgrade.yml
+
+
+- name: install and configure docker
+  include_tasks: install_docker.yml
+
+
+- name: create root container dirs
+  file:
+    path: "{{ item }}"
+    state: directory
+  loop:
+    - "{{ container_config_dir }}"
+    - "{{ container_logs_dir }}"
+    - "{{ container_root_build_dir }}"
+
+
+- name: create child container dirs
+  file:
+    path: "{{ item }}"
+    state: directory
+  loop: "{{ [container_config_dir, container_logs_dir] | product(groups['containers'] | d([])) | map('join', '/') }}"

roles/hv/tasks/upgrade.yml

@@ -0,0 +1,41 @@
+- name: update repository index
+  apt:
+    force_apt_get: yes
+    update_cache: yes
+  changed_when: no
+
+
+- name: ensure apt-show-versions is installed
+  apt:
+    force_apt_get: yes
+    name: apt-show-versions
+    state: latest
+
+
+- name: get upgradeable packages
+  shell:
+    cmd: apt-show-versions --upgradeable
+  register: upgradeable
+  changed_when: no
+
+
+- block:
+  - name: pause and confirm updates
+    pause:
+      prompt: "{{ upgradeable.stdout }}"
+
+
+  - name: upgrade all packages
+    apt:
+      force_apt_get: yes
+      install_recommends: no
+      upgrade: dist
+
+  when: "(upgradeable.stdout_lines is defined) and (upgradeable.stdout_lines | length > 0)"
+
+
+- name: clean repository cache
+  apt:
+    force_apt_get: yes
+    autoclean: yes
+    autoremove: yes

roles/nginx/defaults/main.yml

@@ -0,0 +1,63 @@
+nginx_default_config:
+  root:
+    pcre_jit: yes
+    worker_processes: auto
+    daemon: no
+    error_log: /var/log/nginx/error.log info;
+  events:
+
+  http:
+    aio: threads
+    aio_write: yes
+    directio: 128k
+    sendfile: yes
+    sendfile_max_chunk: 1m
+    tcp_nodelay: yes
+    tcp_nopush: yes
+
+    client_body_buffer_size: 64k
+    client_body_timeout: 30s
+    client_header_buffer_size: 2k
+    client_header_timeout: 15s
+    client_max_body_size: 0
+    send_timeout: 180s
+    resolver_timeout: 10s
+
+    disable_symlinks: yes
+    keepalive_disable: none
+    msie_padding: no
+    server_tokens: no
+    log_not_found: yes
+
+    open_file_cache: 'max=512 inactive=120s'
+    open_file_cache_errors: yes
+
+    gzip: yes
+    gzip_comp_level: 4
+    gzip_min_length: 4096
+    gzip_vary: yes
+    gzip_types:
+      - text/css
+      - text/javascript
+      - text/plain
+      - application/javascript
+      - application/x-javascript
+      - font/truetype
+      - font/opentype
+      - image/svg+xml
+      - application/xml
+
+    autoindex: no
+
+    default_type: application/octet-stream
+
+    proxy_buffer_size: 16k
+    proxy_buffers: '16 16k'
+    proxy_connect_timeout: 30s
+    proxy_http_version: 1.1
+    proxy_read_timeout: 180s
+    proxy_send_timeout: 180s
+    proxy_max_temp_file_size: 0
+
+    http2_push_preload: yes
+

roles/nginx/tasks/prepare_build.yml

@@ -0,0 +1,29 @@
+- name: set nginx_cfg
+  set_fact:
+    nginx_cfg: "{{ nginx_default_config | d({}) | combine(nginx_config | d({}), recursive=true) }}"
+
+
+- name: set container mount dirs
+  set_fact:
+    container_config_mount: /etc/nginx/conf
+    container_logs_mount: /var/log/nginx
+
+
+- name: copy geoip db
+  copy:
+    src: dbip-country-lite.mmdb
+    dest: "{{ (conf_dir, 'geoip.mmdb') | path_join }}"
+
+
+- name: download mime types
+  get_url:
+    url: https://raw.githubusercontent.com/nginx/nginx/master/conf/mime.types
+    dest: "{{ (conf_dir, 'mime.types') | path_join }}"
+
+
+- name: template nginx config
+  template:
+    src: nginx.j2
+    dest: "{{ (conf_dir, 'nginx.conf') | path_join }}"
+    lstrip_blocks: yes
+    force: yes

roles/nginx/templates/nginx.j2

@@ -0,0 +1,85 @@
+{% macro nginx_option(option) -%}
+  {% if option.value is boolean -%}
+    {{ option.key | lower }} {{ 'on' if option.value else 'off' }};
+  {% elif option.value | type_debug == 'list' -%}
+    {{ option.key | lower }} {{ option.value | join(' ') }};
+  {% else -%}
+    {{ option.key | lower }} {{ option.value }};
+  {% endif -%}
+{% endmacro -%}
+
+{% macro nginx_option_block(block) -%}
+  {% if block is mapping -%}
+    {% for option in (block | d({}) | dict2items) -%}
+      {{ nginx_option(option) -}}
+    {% endfor -%}
+  {% endif -%}
+{% endmacro -%}
+
+
+
+
+{{ nginx_option_block(nginx_cfg.root) }}
+
+events {
+  {{ nginx_option_block(nginx_cfg.events) }}
+}
+
+http {
+  {{ nginx_option_block(nginx_cfg.http) }}
+
+  access_log /var/log/nginx/access.log custom;
+  include {{ (container_config_mount, 'mime.types') | path_join | quote}};
+
+  geoip2 {{ (container_config_mount, 'geoip.mmdb') | path_join | quote }} {
+    auto_reload 30m;
+    $geoip_country_code default=RU source=$remote_addr country iso_code;
+  }
+
+  server {
+    listen 80;
+    listen [::]:80;
+
+    server_name apache1.local;
+
+    location / {
+      proxy_pass http://{{ hostvars['apache1']['ansible_host'] }};
+      proxy_set_header Host $proxy_host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Host $server_name;
+    }
+  }
+
+  server {
+    listen 80;
+    listen [::]:80;
+
+    server_name apache2.local;
+
+    location / {
+      proxy_pass http://{{ hostvars['apache2']['ansible_host'] }};
+      proxy_set_header Host $proxy_host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Host $server_name;
+    }
+  }
+
+  server {
+    listen 80 default_server;
+    listen [::]:80 default_server;
+
+    location / {
+      return 404;
+    }
+
+    {% if 'nginx_exporter' in groups['all'] -%}
+    location /stub_status {
+      stub_status;
+      # deny all;
+      allow {{ hostvars['nginx_exporter']['ansible_host'] }};
+    }
+    {% endif -%}
+  }
+}

roles/nginx_exporter/tasks/prepare_build.yml

@@ -0,0 +1,4 @@
+- name: set container cfg
+  set_fact:
+    container_role_config:
+      command: "-nginx.scrape-uri=http://{{ hostvars['nginx']['ansible_host'] }}/stub_status"

templates/nginx.Dockerfile.j2

@@ -0,0 +1,50 @@
+FROM nginx:alpine AS builder
+
+RUN apk add --no-cache --virtual .build-deps \
+    gcc \
+    geoip-dev \
+    git \
+    libc-dev \
+    libmaxminddb \
+    libmaxminddb-dev \
+    make \
+    openssl-dev \
+    pcre-dev \
+    python3-dev \
+    py3-pip \
+    zlib-dev
+
+WORKDIR /usr/local
+
+RUN git clone https://github.com/leev/ngx_http_geoip2_module.git --depth=1
+
+RUN wget http://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz && \
+    mkdir -p /usr/src && \
+    tar -zxC /usr/src -f nginx-${NGINX_VERSION}.tar.gz
+
+RUN CONFARGS=$(nginx -V 2>&1 | sed -n -e 's/^.*arguments: //p') \
+    cd /usr/src/nginx-$NGINX_VERSION && \
+    ./configure --with-compat $CONFARGS \
+    --add-dynamic-module=/usr/local/ngx_http_geoip2_module/ && \
+    make && \
+    make modules && \
+    make install && \
+    mkdir -p /usr/local/nginx/modules/
+
+
+FROM nginx:alpine
+COPY --from=builder /usr/local/nginx/modules/ngx_http_geoip2_module.so /usr/local/nginx/modules/ngx_http_geoip2_module.so
+
+RUN apk add \
+    libmaxminddb \
+    libmaxminddb-dev
+
+RUN mkdir -p /etc/nginx/conf
+
+WORKDIR /usr/share/nginx/html
+
+EXPOSE 80 443
+
+VOLUME [ "/usr/share/nginx/html" ]
+
+ENTRYPOINT [ "/usr/sbin/nginx", "-c", "/etc/nginx/conf/nginx.conf" ]