feat: add mysql/mysql_exporter/fluentd

README.md

@@ -2,6 +2,7 @@
 
 ## Пререквизиты
 - ВМ или контейнер с Debian 11, который будет работать в качестве Docker-хоста (Docker ставить не нужно)
+- Ansible-контроллер под Alpine или Debian
 - настроено SSH-подключение под учеткой root (для Ansible)
 - Python 3 (для Ansible)
 - две записи в файле hosts:
@@ -34,12 +35,12 @@ docker_network_iprange: 10.90.0.0/16
 - [x] apache2
 - [x] node_exporter
 - [x] nginx_exporter
-- [ ] mysql_exporter
+- [x] mysql_exporter
 - [x] cadvisor
 - [ ] prometheus
 - [ ] grafana
-- [ ] fluentd
+- [x] fluentd
-- [ ] mysql
+- [x] mysql
 
 ## Исходные данные
 

all.yml

@@ -1,2 +1,3 @@
+- import_playbook: ansible.yml
 - import_playbook: hv.yml
 - import_playbook: containers.yml

ansible.yml

@@ -0,0 +1,6 @@
+- hosts: localhost
+  gather_facts: no
+  serial: 1
+  strategy: linear
+  roles:
+    - ansible

group_vars/all.yml

@@ -3,3 +3,12 @@ container_logs_dir: /opt/logs
 container_root_build_dir: /opt/build
 
 docker_remote_port: 5011
+
+fluentd_database_name: nginx
+fluentd_database_user: fluentd
+fluentd_database_pass: 4CDVyJQPGsrzmy3U3XgdYEKGGxoRBquy
+fluentd_database_table: nginx_logs
+
+
+gpg_keyrings_dir: /etc/apt/trusted.gpg.d
+docker_repo_url: https://download.docker.com/linux/debian

group_vars/containers.yml

@@ -1,4 +1,5 @@
-ansible_connection: docker
 hypervisor_hostname: hv
 
-container_default_config:
+ansible_connection: docker
+ansible_user: root
+ansible_docker_extra_args: "-H tcp://{{ hostvars[hypervisor_hostname]['ansible_host'] ~ ':' ~ docker_remote_port }}"

hosts

@@ -10,8 +10,7 @@ all:
     containers:
       hosts:
         nginx:
-          ansible_host: 10.90.0.2
+          container_address: 10.90.0.2
-          ansible_password: k3PpKHF52zgUaowB73V6ggrnMUDUcMeS
           container_config:
             image: nginx
             custom_image: yes
@@ -20,29 +19,42 @@ all:
               error_log: /var/log/nginx/error.log info
               load_module: /usr/local/nginx/modules/ngx_http_geoip2_module.so
             http:
-              log_format: "custom escape=json '\"$time_iso8601\" \"$request_time\" \"$upstream_response_time\" \
+              log_format: "custom escape=json '{\
-                           \"$remote_addr\" \"$remote_user\" \"$time_local\" \"$request\" \"$status\" \
+                           \"time_iso8601\": \"$time_iso8601\", \
-                           \"$body_bytes_sent\" \"$http_referer\" \"$http_user_agent\" \"$geoip_country_code\"'"
+                           \"request_time\": \"$request_time\", \
+                           \"upstream_response_time\": \"$upstream_response_time\", \
+                           \"remote_addr\": \"$remote_addr\", \
+                           \"remote_user\": \"$remote_user\", \
+                           \"time_local\": \"$time_local\", \
+                           \"request\": \"$request\", \
+                           \"status\": \"$status\", \
+                           \"body_bytes_sent\": \"$body_bytes_sent\", \
+                           \"http_referer\": \"$http_referer\", \
+                           \"http_user_agent\": \"$http_user_agent\", \
+                           \"geoip_country_code\": \"$geoip_country_code\"}'"
+            http_post:
+              access_log: "syslog:server=10.90.0.10:11514,tag=nginx_access custom"
+          fluentd_source:
+            _type: syslog
+            port: 11514
+            tag: nginx.access
 
         apache1:
-          ansible_host: 10.90.0.3
+          container_address: 10.90.0.3
-          ansible_password: qUTuyFHzCFGVswGYEHv5MU2JzQGt9Tx7
           host_role: apache
           container_config:
             image: php:8-apache
             mounts: /opt/www/html:/var/www/html
 
         apache2:
-          ansible_host: 10.90.0.4
+          container_address: 10.90.0.4
-          ansible_password: F2a4v4LoQ5U6rwAgsrSt68SJwmHGARuP
           host_role: apache
           container_config:
             image: php:8-apache
             mounts: /opt/www/html:/var/www/html
 
         cadvisor:
-          ansible_host: 10.90.0.5
+          container_address: 10.90.0.5
-          ansible_password: dJpxV9gri438UbC82bYXCp4BeAGHyoZn
           container_config:
             image: gcr.io/cadvisor/cadvisor:latest
             mounts:
@@ -56,8 +68,7 @@ all:
               - /dev/kmsg
 
         node_exporter:
-          ansible_host: 10.90.0.6
+          container_address: 10.90.0.6
-          ansible_password: QnSKHZ5e82YnCzbS75PMHXZqhwpFbzcu
           container_config:
             image: prom/node-exporter:latest
             extra_mounts:
@@ -65,23 +76,36 @@ all:
             command: '--path.rootfs=/host'
 
         nginx_exporter:
-          ansible_host: 10.90.0.7
+          container_address: 10.90.0.7
-          ansible_password: rkeE2vHp2jmLjsyz4r9ASe63GM5t6FbA
           container_config:
             image: nginx/nginx-prometheus-exporter
 
+        mysql:
+          container_address: 10.90.0.8
+          container_config:
+            image: yobasystems/alpine-mariadb:latest
+            mounts: /opt/mariadb:/var/lib/mysql
+            command: '--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci'
+          mysql_root_password: eJdr5qyXoH2LCna3FaL9w3pAzpxGdWSE
+
         mysql_exporter:
-          ansible_host: 10.61.1.7
+          container_address: 10.90.0.9
-          ansible_password: VxMNsdHg6Fu5Ah9GxRTcQFVTGFRagAHD
+          container_config:
+            image: prom/mysqld-exporter:latest
+          mysql_target_host: mysql
+          mysql_exporter_username: exporter
+          mysql_exporter_password: JmH45F3HQogkQoofLaca7YNPvYRV4KXD
+
+        fluentd:
+          container_address: 10.90.0.10
+          container_config:
+            image: fluentd
+            custom_image: yes
+          fluentd_syslog_port: 11514
+
+
+
         prometheus:
-          ansible_host: 10.61.1.9
+          container_address: 10.90.0.11
-          ansible_password: XPJEzq3ohbu3KxcqWnT65M9uX5Utxyx2
         grafana:
-          ansible_host: 10.61.1.10
+          container_address: 10.90.0.12
-          ansible_password: ZsA8if5Cm4sEr299SjoMEMgqV5kBYn4a
-        fluentd:
-          ansible_host: 10.61.1.11
-          ansible_password: mcLjXRpVGvxk7x4NPhuHVf9n6o8nk3cS
-        mysql:
-          ansible_host: 10.61.1.12
-          ansible_password: cr9GvFGoqnzAdUNRvTr2DRqwQqqKLMWV

roles/ansible/tasks/main.yml

@@ -0,0 +1,25 @@
+- name: determine distribution
+  setup:
+    gather_subset:
+      - distribution
+
+
+- name: install docker cli
+  package:
+    name: docker-cli
+  when: ansible_distribution == 'Alpine'
+
+
+- block:
+    - name: add docker repository
+      include_tasks: add_docker_repo.yml
+
+
+    - name: install docker cli
+      apt:
+        name:
+          - docker-ce-cli
+        force_apt_get: yes
+        state: latest
+
+  when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'

roles/container/tasks/main.yml

@@ -79,9 +79,7 @@
         hostname: "{{ inventory_hostname }}"
         command_handling: correct
         network_mode: bridge
-        networks:
+        networks: "{{ [] if container_address is not defined else [{'name': 'network', 'ipv4_address': container_address}] }}"
-          - name: network
-            ipv4_address: "{{ ansible_host }}"
         log_driver: local
         detach: yes
         restart_policy: unless-stopped
@@ -91,5 +89,18 @@
         privileged: "{{ container_cfg.privileged | d(false) }}"
         devices: "{{ omit if container_cfg.devices is not defined else ([container_cfg.devices] | flatten(levels=1)) }}"
         command: "{{ container_cfg.command | d(omit) }}"
+        env: "{{ container_cfg.env | d(omit) }}"
+        exposed_ports: "{{ container_cfg.exposed_ports | d(omit) }}"
 
   delegate_to: "{{ hypervisor_hostname }}"
+
+
+- block:
+    - name: ping container
+      ping:
+
+  rescue:
+    - name: install python in container
+      raw: apk add python3
+
+  when: container_post_build | d(false) == true

roles/fluentd/tasks/prepare_build.yml

@@ -0,0 +1,118 @@
+- name: set container mount dirs
+  set_fact:
+    container_config_mount: /fluentd/etc/
+
+
+- name: set additional exposed ports
+  set_fact:
+    container_role_config:
+      exposed_ports: "{{ fluentd_syslog_port }}"
+  when: fluentd_syslog_port is defined
+
+
+- name: set fluentd sections
+  set_fact:
+    fluentd_filters:
+      - __postfix: 'nginx.**'
+        _type: parser
+        key_name: message
+        parse:
+          _type: json
+          json_parser: yajl
+
+    fluentd_matches:
+      - __postfix: 'nginx.**'
+        _type: sql
+        adapter: mysql2
+        _log_level: trace
+        port: 3306
+        host: "{{ hostvars['mysql']['container_address'] | d(hostvars['mysql']['ansible_host']) }}"
+        database: "{{ fluentd_database_name }}"
+        username: "{{ fluentd_database_user }}"
+        password: "{{ fluentd_database_pass }}"
+        flush_interval: 1s
+        table:
+          table: "{{ fluentd_database_table }}"
+          column_mapping: "time_iso8601:time_iso8601,\
+                           request_time:request_time,\
+                           upstream_response_time:upstream_response_time,\
+                           remote_addr:remote_addr,\
+                           remote_user:remote_user,\
+                           time_local:time_local,\
+                           request:request,\
+                           status:status,\
+                           body_bytes_sent:body_bytes_sent,\
+                           http_referer:http_referer,\
+                           http_user_agent:http_user_agent,\
+                           geoip_country_code:geoip_country_code"
+
+
+- block:
+    - name: initialize ansible mysql connection
+      include_tasks: init_mysql.yml
+
+
+    - name: create fluentd database
+      mysql_db:
+        name: "{{ fluentd_database_name }}"
+        encoding: utf8
+        login_host: "{{ hostvars['mysql']['container_address'] | d(hostvars['mysql']['ansible_host']) }}"
+        login_user: "{{ hostvars['mysql']['mysql_root_username'] | d('root') }}"
+        login_password: "{{ hostvars['mysql']['mysql_root_password'] | d(omit) }}"
+
+
+    - name: create fluentd database user
+      mysql_user:
+        name: "{{ fluentd_database_user }}"
+        password: "{{ fluentd_database_pass }}"
+        plugin: mysql_native_password
+        host: "{{ container_address | d(ansible_host) }}"
+        priv: "{{ fluentd_database_name }}.*:ALL"
+
+        login_host: "{{ hostvars['mysql']['container_address'] | d(hostvars['mysql']['ansible_host']) }}"
+        login_user: "{{ hostvars['mysql']['mysql_root_username'] | d('root') }}"
+        login_password: "{{ hostvars['mysql']['mysql_root_password'] | d(omit) }}"
+      register: result
+
+
+    - name: flush privileges
+      mysql_query:
+        query: FLUSH PRIVILEGES
+        login_host: "{{ hostvars['mysql']['container_address'] | d(hostvars['mysql']['ansible_host']) }}"
+        login_user: "{{ hostvars['mysql']['mysql_root_username'] | d('root') }}"
+        login_password: "{{ hostvars['mysql']['mysql_root_password'] | d(omit) }}"
+      when: result.changed
+
+
+    - name: initialize mysql database
+      mysql_query:
+        query: |
+          CREATE TABLE IF NOT EXISTS {{ fluentd_database_table | quote }} (
+            time_iso8601 DATETIME,
+            request_time FLOAT,
+            upstream_response_time FLOAT,
+            remote_addr TEXT,
+            remote_user TEXT,
+            time_local TEXT,
+            request TEXT,
+            status INTEGER,
+            body_bytes_sent INTEGER,
+            http_referer TEXT,
+            http_user_agent TEXT,
+            geoip_country_code TEXT
+          );
+        login_db: "{{ fluentd_database_name }}"
+        login_host: "{{ hostvars['mysql']['container_address'] | d(hostvars['mysql']['ansible_host']) }}"
+        login_user: "{{ hostvars['mysql']['mysql_root_username'] | d('root') }}"
+        login_password: "{{ hostvars['mysql']['mysql_root_password'] | d(omit) }}"
+      changed_when: no
+
+  delegate_to: localhost
+
+
+- name: template fluentd config
+  template:
+    src: fluentd.j2
+    dest: "{{ (conf_dir, 'fluent.conf') | path_join }}"
+    lstrip_blocks: yes
+    force: yes

roles/fluentd/templates/fluentd.j2

@@ -0,0 +1,41 @@
+{% macro fluentd_option_block(name, values, padding = 2) -%}
+  {% if values is mapping -%}
+    {% set postfix = '' if values.__postfix is not defined else (' ' + values.__postfix) -%}
+
+    <{{ name ~ postfix}}>
+      {% for option in values | dict2items -%}
+        {% if option.key != '__postfix' -%}
+          {{ '' if padding == 0 else (' ' * padding) -}}
+          {% if option.value is mapping -%}
+            {{ fluentd_option_block(option.key, option.value, padding + 2) -}}
+          {% else -%}
+            {% if option.key[0] == '_' -%}
+              {{ '@' ~ option.key[1:] }} {{ option.value }}
+            {% else -%}
+              {{ option.key }} {{ option.value }}
+            {% endif -%}
+          {% endif -%}
+        {% endif -%}
+      {% endfor -%}
+    {{ '' if padding == 0 else (' ' * (padding - 2)) -}}</{{ name }}>
+  {% endif -%}
+{% endmacro -%}
+
+
+
+{{ fluentd_option_block('system', fluentd_system | d(None)) }}
+
+
+{% for source in (fluentd_sources | d([])) + (groups['all'] | map('extract', hostvars) | selectattr('fluentd_source', 'mapping') | map(attribute='fluentd_source') | list) -%}
+  {{ fluentd_option_block('source', source) }}
+{% endfor -%}
+
+
+{% for filter in (fluentd_filters | d([])) -%}
+  {{ fluentd_option_block('filter', filter) }}
+{% endfor -%}
+
+
+{% for match in (fluentd_matches | d([])) -%}
+  {{ fluentd_option_block('match', match) }}
+{% endfor -%}

roles/hv/defaults/main.yml

@@ -1,7 +1,4 @@
 timezone: Europe/Kirov
-gpg_keyrings_dir: /etc/apt/trusted.gpg.d
-docker_repo_url: https://download.docker.com/linux/debian
-
 
 docker_network_subnet: 10.90.0.0/16
 docker_network_gateway: 10.90.0.1

roles/hv/tasks/install_docker.yml

@@ -13,41 +13,8 @@
     state: absent
 
 
-- name: ensure apt can access https repos
+- name: add docker repository
-  apt:
+  include_tasks: add_docker_repo.yml
-    name:
-      - ca-certificates
-      - curl
-      - gnupg
-      - lsb-release
-    force_apt_get: yes
-    state: latest
-
-
-- name: add keyrings dir
-  file:
-    state: directory
-    path: "{{ gpg_keyrings_dir }}"
-
-
-- name: download docker gpg key
-  get_url:
-    url: "{{ docker_repo_url ~ '/gpg' }}"
-    dest: "{{ gpg_keyrings_dir }}/docker.asc"
-    mode: a+r
-
-
-- name: add apt repo
-  apt_repository:
-    repo: "deb [arch=amd64 signed-by={{ (gpg_keyrings_dir ~ '/docker.asc') | quote }}] \
-           {{ docker_repo_url }} {{ ansible_distribution_release }} stable"
-
-
-- name: update repository index
-  apt:
-    force_apt_get: yes
-    update_cache: yes
-  changed_when: no
 
 
 - name: install docker and dependencies

roles/mysql/tasks/prepare_build.yml

@@ -0,0 +1,12 @@
+- name: set container cfg
+  set_fact:
+    container_logs_mount: /var/lib/mysql/mysql-bin
+    container_role_config:
+      env:
+        MYSQL_ROOT_PASSWORD: "{{ mysql_root_password }}"
+
+
+- name: create mysql dir
+  file:
+    path: /opt/mariadb
+    state: directory

roles/mysql_exporter/tasks/main.yml

@@ -0,0 +1,37 @@
+- name: fail if exporter host is missing
+  fail:
+    msg: mysql_target_host is not string
+  when: mysql_target_host is not string
+
+
+- block:
+    - include_tasks: init_mysql.yml
+
+
+    - name: create exporter user
+      mysql_user:
+        name: "{{ mysql_exporter_username }}"
+        password: "{{ mysql_exporter_password }}"
+        plugin: mysql_native_password
+        host: "{{ container_address | d(ansible_host) }}"
+        resource_limits:
+          MAX_USER_CONNECTIONS: 3
+        priv:
+          '*.*': 'SELECT,BINLOG MONITOR,PROCESS'
+          'performance_schema.*': SELECT
+
+        login_host: "{{ hostvars[mysql_target_host]['container_address'] | d(hostvars[mysql_target_host]['ansible_host']) }}"
+        login_user: "{{ hostvars[mysql_target_host]['mysql_root_username'] | d('root') }}"
+        login_password: "{{ hostvars[mysql_target_host]['mysql_root_password'] | d(omit) }}"
+      register: result
+
+
+    - name: flush privileges
+      mysql_query:
+        query: FLUSH PRIVILEGES
+        login_host: "{{ hostvars[mysql_target_host]['container_address'] | d(hostvars[mysql_target_host]['ansible_host']) }}"
+        login_user: "{{ hostvars[mysql_target_host]['mysql_root_username'] | d('root') }}"
+        login_password: "{{ hostvars[mysql_target_host]['mysql_root_password'] | d(omit) }}"
+      when: result.changed
+
+  delegate_to: localhost

roles/mysql_exporter/tasks/prepare_build.yml

@@ -0,0 +1,7 @@
+- name: set container cfg
+  set_fact:
+    container_role_config:
+      env:
+        DATA_SOURCE_NAME: "{{ mysql_exporter_username ~ ':' ~ mysql_exporter_password ~ '@tcp(' ~
+                              (hostvars[mysql_target_host]['container_address'] | d(hostvars[mysql_target_host]['ansible_host'])) ~
+                              ':3306)/' }}"

roles/nginx/templates/nginx.j2

@@ -28,7 +28,6 @@ events {
 http {
   {{ nginx_option_block(nginx_cfg.http) }}
 
-  access_log /var/log/nginx/access.log custom;
   include {{ (container_config_mount, 'mime.types') | path_join | quote}};
 
   geoip2 {{ (container_config_mount, 'geoip.mmdb') | path_join | quote }} {
@@ -36,6 +35,8 @@ http {
     $geoip_country_code default=RU source=$remote_addr country iso_code;
   }
 
+  {{ nginx_option_block(nginx_cfg.http_post) }}
+
   server {
     listen 80;
     listen [::]:80;
@@ -43,7 +44,7 @@ http {
     server_name apache1.local;
 
     location / {
-      proxy_pass http://{{ hostvars['apache1']['ansible_host'] }};
+      proxy_pass http://{{ hostvars['apache1']['container_address'] | d(hostvars['apache1']['ansible_host']) }};
       proxy_set_header Host $proxy_host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -58,7 +59,7 @@ http {
     server_name apache2.local;
 
     location / {
-      proxy_pass http://{{ hostvars['apache2']['ansible_host'] }};
+      proxy_pass http://{{ hostvars['apache2']['container_address'] | d(hostvars['apache2']['ansible_host']) }};
       proxy_set_header Host $proxy_host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -78,7 +79,7 @@ http {
     location /stub_status {
       stub_status;
       # deny all;
-      allow {{ hostvars['nginx_exporter']['ansible_host'] }};
+      allow {{ hostvars['nginx_exporter']['container_address'] | d(hostvars['nginx_exporter']['ansible_host']) }};
     }
     {% endif -%}
   }

roles/nginx_exporter/tasks/prepare_build.yml

@@ -1,4 +1,4 @@
 - name: set container cfg
   set_fact:
     container_role_config:
-      command: "-nginx.scrape-uri=http://{{ hostvars['nginx']['ansible_host'] }}/stub_status"
+      command: "-nginx.scrape-uri=http://{{ hostvars['nginx']['container_address'] | d(hostvars['nginx']['ansible_host']) }}/stub_status"

tasks/add_docker_repo.yml

@@ -0,0 +1,44 @@
+- name: determine distribution
+  setup:
+    gather_subset:
+      - distribution
+
+
+- block:
+    - name: ensure apt can access https repos
+      apt:
+        name:
+          - ca-certificates
+          - curl
+          - gnupg
+          - lsb-release
+        force_apt_get: yes
+        state: latest
+
+
+    - name: add keyrings dir
+      file:
+        state: directory
+        path: "{{ gpg_keyrings_dir }}"
+
+
+    - name: download docker gpg key
+      get_url:
+        url: "{{ docker_repo_url ~ '/gpg' }}"
+        dest: "{{ gpg_keyrings_dir }}/docker.asc"
+        mode: a+r
+
+
+    - name: add apt repo
+      apt_repository:
+        repo: "deb [arch=amd64 signed-by={{ (gpg_keyrings_dir ~ '/docker.asc') | quote }}] \
+               {{ docker_repo_url }} {{ ansible_distribution_release }} stable"
+
+
+    - name: update repository index
+      apt:
+        force_apt_get: yes
+        update_cache: yes
+      changed_when: no
+
+  when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'

tasks/init_mysql.yml

@@ -0,0 +1,13 @@
+- name: initialize mysql connection on ansible controller
+  block:
+    - name: get ansible controller os distribution
+      setup:
+        gather_subset:
+          - distribution
+
+
+    - name: install pymysql
+      package:
+        name: "{{ { 'alpine': 'py3-pymysql', 'debian': 'python3-pymysql' }[ansible_distribution | lower] | d('pymysql') }}"
+
+  delegate_to: localhost

templates/fluentd.Dockerfile.j2

@@ -0,0 +1,14 @@
+FROM fluent/fluentd:edge
+
+USER root
+
+RUN apk add --no-cache --update --virtual .build-deps build-base ruby-dev && \
+        apk add mariadb-dev mariadb-connector-c mariadb-connector-c-dev && \
+        gem install mysql2 && \
+        gem install fluent-plugin-record-modifier && \
+        gem install fluent-plugin-sql && \
+        gem sources --clear-all && \
+        apk del .build-deps && \
+        rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
+
+USER fluent