feat(deployment): add Prometheus support for monitoring

Closes #337

Author: michaelfig

packages/deployment/README.md

@@ -0,0 +1,11 @@
+# Agoric testnet Deployment
+
+**NOTE: This private package does not create secure public testnets by default.  We recommend that public validators use their own well-understood, diversified means of deployment.**
+
+You can use this package to configure privately-accessible testnets, such as on your local workstation's Docker, or a local-to-your organization LAN.
+
+If you want to create a publically-accessible testnet you will at least need to:
+
+1. Configure your nodes' firewalls so that only the necessary ports are accessible.
+2. Set up your provisioning server (node0) to expose the provisioner only on HTTPS with some form of authentication.
+3. **Do more, which we are not sure of**

packages/deployment/ansible/install-cosmos.yml

@@ -8,6 +8,7 @@
   vars:
     - service: ag-chain-cosmos
     - data: "{{ SETUP_HOME }}/{{ service }}/data"
+    - execline: "/usr/src/cosmic-swingset/lib/ag-chain-cosmos start --proxy_app=kvstore"
     - PERSISTENT_PEERS: "{{ lookup('file', SETUP_HOME + '/' + service + '/data/peers.txt') }}"
     - NUM_FILE_DESCRIPTORS: 2048
   roles:

packages/deployment/ansible/prepare-machine.yml

@@ -6,7 +6,6 @@
   gather_facts: yes
   strategy: free
   vars:
-    - service: ag-chain-cosmos
     - NODEJS_VERSION: 12
   roles:
     - prereq

packages/deployment/ansible/prereq.yml

@@ -0,0 +1,14 @@
+---
+
+- hosts: all
+  user: root
+  gather_facts: no
+  strategy: free
+  vars:
+    - service: node-exporter
+    - user: root
+    - to_remove:
+      - /etc/systemd/system/node-exporter.service
+  roles:
+    - stop
+    - remove

packages/deployment/ansible/prometheus-node-disable.yml

@@ -0,0 +1,17 @@
+---
+
+- hosts: all
+  user: root
+  gather_facts: no
+  strategy: free
+  vars:
+    - PROM_NODE_EXPORTER_VERSION: 0.18.1
+    - service: node-exporter
+    - execline: /usr/local/sbin/node_exporter
+    - chdir: /
+    - user: root
+  roles:
+    - prometheus-prereq
+    - install
+    - stop
+    - start

packages/deployment/ansible/prometheus-node-enable.yml

@@ -22,6 +22,36 @@
     regexp: '^addr_book_strict *='
     line: 'addr_book_strict = false'
 
+- name: "check if prometheus-tendermint.txt exists"
+  delegate_to: localhost
+  stat:
+    path: "{{ SETUP_HOME }}/prometheus-tendermint.txt"
+  register: "prommint"
+
+# NOTE: This is protected by the server firewall
+#- name: Set prometheus_listen_addr=127.0.0.1:26660
+#  lineinfile:
+#    path: "/home/{{ service }}/.{{ service }}/config/config.toml"
+#    state: present
+#    regexp: '^prometheus_listen_addr *='
+#    line: 'prometheus_listen_addr = "127.0.0.1:26660"'
+
+- name: Set prometheus=true
+  lineinfile:
+    path: "/home/{{ service }}/.{{ service }}/config/config.toml"
+    state: present
+    regexp: '^prometheus *='
+    line: 'prometheus = true'
+  when: prommint.stat.exists
+
+- name: Set prometheus=false
+  lineinfile:
+    path: "/home/{{ service }}/.{{ service }}/config/config.toml"
+    state: present
+    regexp: '^prometheus *='
+    line: 'prometheus = false'
+  when: not prommint.stat.exists
+
 - name: Listen for public RPC
   lineinfile:
     path: "/home/{{ service }}/.{{ service }}/config/config.toml"

packages/deployment/ansible/roles/install-cosmos/tasks/main.yml

@@ -4,6 +4,4 @@
   template:
     src: systemd.service.j2
     dest: "/etc/systemd/system/{{service}}.service"
-  vars:
-    execline: "/usr/src/cosmic-swingset/lib/ag-chain-cosmos start --proxy_app=kvstore"
   notify: reload services

packages/deployment/ansible/roles/install/tasks/main.yml

@@ -0,0 +1,18 @@
+- name: Download Prometheus exporter
+  delegate_to: localhost
+  get_url:
+    url: "https://github.com/prometheus/node_exporter/releases/download/v0.18.1/node_exporter-{{ PROM_NODE_EXPORTER_VERSION }}.linux-amd64.tar.gz"
+    dest: "/root/node_exporter{{ PROM_NODE_EXPORTER_VERSION }}.tgz"
+
+- name: Extract Prometheus exporter
+  delegate_to: localhost
+  unarchive:
+    src: "/root/node_exporter{{ PROM_NODE_EXPORTER_VERSION }}.tgz"
+    dest: "/root"
+    creates: "/root/node_exporter-{{ PROM_NODE_EXPORTER_VERSION }}.linux-amd64/node_exporter"
+
+- name: Install Prometheus exporter
+  copy:
+    src: "/root/node_exporter-{{ PROM_NODE_EXPORTER_VERSION }}.linux-amd64/node_exporter"
+    dest: "/usr/local/sbin/node_exporter"
+    mode: 0755

packages/deployment/ansible/roles/prometheus-prereq/tasks/main.yml

@@ -0,0 +1,9 @@
+* to configure Prometheus endpoints on your deployed testnet nodes:
+  1. for validator (i.e. Tendermint) on TCP port 26660 /metrics
+    - to enable: `date | ag-chain-cosmos shell -c 'tee MYTESTNET/prometheus-tendermint.txt'`
+      - takes effect on next `NETWORK_NAME=MYTESTNET ag-chain-cosmos bootstrap --bump`
+    - to disable: `ag-chain-cosmos shell -c 'rm MYTESTNET/prometheus-tendermint.txt'`
+      - takes effect on next `NETWORK_NAME=MYTESTNET ag-chain-cosmos bootstrap --bump`
+  2. for host-level (i.e. node_exporter) on TCP port 9100 /metrics
+    - to enable: `ag-chain-cosmos play prometheus-node-enable`
+    - to disable: `ag-chain-cosmos play prometheus-node-disable`

packages/deployment/changelogs/337.txt

@@ -0,0 +1,13 @@
+For each PR, add a file to this directory named $ISSUENUMBER.txt , and
+describe any downstream-visible changes in it (one per line). For libraries,
+this should include anything a developer using this library needs to know
+when they upgrade to the new version (API changes, new features, significant
+bugs fixed). If the PR only makes internal changes (refactorings,
+documentation updates), you should still add a file, but leave it empty.
+
+These files will be concatenated together and added to the NEWS.md file
+during the release process. Their filenames will be used to indicate which
+issues were closed in the release.
+
+See the top-level developer docs for more details.
+

packages/deployment/changelogs/README-changelogs.md

@@ -402,6 +402,7 @@ show-config      display the client connection parameters
         needReMain([
           'play',
           'install',
+          `-eexecline=${shellEscape('/usr/src/cosmic-swingset/lib/ag-chain-cosmos start --proxy_app=kvstore')}`,
           `-eserviceLines="Environment=BOOT_ADDRESS=${bootAddress}"`,
         ]),
       );