Update Release Candidate

Author: perrinjerome

software/erp5/instance-erp5-input-schema.json

@@ -798,7 +798,7 @@
         },
         "apachedex-promise-threshold": {
           "type": "number",
-          "title": "Promise fails if the overall Apdex score for the previous day is below than this value.",
+          "title": "Promise fails if the overall Apdex score for the previous day is below than this value. A value of 0 disables the promise.",
           "default": 70,
           "minimum": 0,
           "maximum": 100

software/erp5/test/test/test_balancer.py

@@ -1,6 +1,7 @@
 import ipaddress
 import json
 import logging
+import lzma
 import os
 import re
 import socket
@@ -104,7 +105,7 @@ def _getInstanceParameterDict(cls) -> dict:
             '--js-embed',
             '--quiet',
           ],
-        'apachedex-promise-threshold': 100,
+        'apachedex-promise-threshold': 0,
         'haproxy-server-check-path': '/',
         'zope-family-dict': {
             'default': ['dummy_http_server'],
@@ -264,7 +265,7 @@ def test_access_log_apachedex_report(self) -> None:
     # make a request so that we have something in the logs
     requests.get(self.default_balancer_zope_url, verify=False)
 
-    # crontab for apachedex is executed
+    # crontab for daily apachedex is executed
     self._executeCrontabAtDate('generate-apachedex-report', '23:59')
     # it creates a report for the day
     apachedex_report, = (
@@ -278,6 +279,39 @@ def test_access_log_apachedex_report(self) -> None:
     # having this table means that apachedex could parse some lines.
     self.assertIn('<h2>Hits per status code</h2>', report_text)
 
+    # weekly apachedex uses the logs after rotation, we'll run log rotation
+    # until we have a xz file for two days ago and a non compressed file for
+    # yesterday
+    # run logrotate a first time so that it create state files
+    self._executeCrontabAtDate('logrotate', '2000-01-01')
+    requests.get(urllib.parse.urljoin(self.default_balancer_zope_url, 'error-two-days-ago'), verify=False)
+    self._executeCrontabAtDate('logrotate', 'yesterday 00:00')
+    requests.get(urllib.parse.urljoin(self.default_balancer_zope_url, 'error-yesterday'), verify=False)
+    self._executeCrontabAtDate('logrotate', '00:00')
+
+    # this apachedex command uses compressed files, verify that our test setup
+    # is correct and that the error from two days ago is in the compressed file.
+    two_days_ago_log, = (
+      self.computer_partition_root_path / 'srv' / 'backup'/ 'logrotate'
+    ).glob("apache-access.log-*.xz")
+    with lzma.open(two_days_ago_log) as f:
+      self.assertIn(b'GET /error-two-days-ago', f.read())
+
+    self._executeCrontabAtDate('generate-weekly-apachedex-report', '23:59')
+    # this creates a report for the week
+    apachedex_weekly_report, = (
+      self.computer_partition_root_path
+        / 'srv'
+        / 'monitor'
+        / 'private'
+        / 'apachedex'
+        / 'weekly').glob('*.html')
+    weekly_report_text = apachedex_weekly_report.read_text()
+    self.assertIn('APacheDEX', weekly_report_text)
+    # because we run apachedex with error details, we can see our error requests
+    self.assertIn('error-two-days-ago', weekly_report_text)
+    self.assertIn('error-yesterday', weekly_report_text)
+
   def test_access_log_rotation(self) -> None:
     # run logrotate a first time so that it create state files
     self._executeCrontabAtDate('logrotate', '2000-01-01')

software/neoppod/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = b4baf7f21f450fa522c2a69f5a4aedf7
 
 [root-common]
 filename = root-common.cfg.in
-md5sum = eefc3358852b7509bfe1b18da19abca0
+md5sum = 1320dbea362ce6909f1490ba349c3e6c
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
 
 [instance-neo]
 filename = instance-neo.cfg.in
-md5sum = 53b2c73a0c5c7cb5f6b841ba6892fa46
+md5sum = d8a62f4a2b68fe97146871d07a500443
 
 [template-neo-my-cnf]
 filename = my.cnf.in

software/neoppod/instance-neo-input-schema.json

@@ -28,6 +28,49 @@
           "description": "Master nodes in the cluster to backup.",
           "type": "string"
         },
+        "reflink": {
+          "description": "Track references between OIDs of an external ZODB.",
+          "additionalProperties": false,
+          "required": [
+            "zurl"
+          ],
+          "properties": {
+            "zurl": {
+              "description": "ZODB to track.",
+              "type": "string"
+            },
+            "_ca": {
+              "type": "string"
+            },
+            "_cert": {
+              "type": "string"
+            },
+            "_key": {
+              "type": "string"
+            },
+            "max-txn-size": {
+              "description": "Maximum number of OIDs to delete per transaction.",
+              "default": 2097151,
+              "type": "integer"
+            },
+            "commit-interval": {
+              "description": "Commit every SECONDS of work.",
+              "default": 10,
+              "type": "integer"
+            },
+            "period": {
+              "description": "Age of the historical revision at which a GC is performed (this can be seen as a grace period). See --period option.",
+              "default": 86400,
+              "type": "integer"
+            },
+            "no-gc": {
+              "description": "Only track references.",
+              "default": false,
+              "type": "boolean"
+            },
+          },
+          "type": "object"
+        },
         "monitor": {
           "description": "Parameters for monitoring.",
           "properties": {
@@ -124,7 +167,7 @@
               "type": "array"
             },
             "storage-type": {
-              "description": "Storage type. Required when several types are configured and you select which one to use via 'node!' parameter. Defaults to whatever is configured ('sqlite' or 'mysql'), else MySQL if available, else SQLite.",
+              "description": "Storage type. Required when several types are configured and you select which one to use via 'node!' parameter. Defaults to whatever is configured ('sqlite' or 'mysql'), else MySQL if available, else SQLite. SQLite is recommended for a reflink DB.",
               "enum": [
                 "MySQL",
                 "SQLite"

software/neoppod/instance-neo.cfg.in

@@ -194,6 +194,44 @@ post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- {{ bin_directory }}/neostor
 
 {% endfor -%}
 
+{% set reflink = slapparameter_dict.get('reflink') -%}
+{% if reflink -%}
+{%   set zurl = 'neo://%s@%s' % (slapparameter_dict['cluster'], slapparameter_dict['masters'].replace(' ', ',')) -%}
+{%   if slapparameter_dict['ssl'] -%}
+{%     set zurl = zurl + '?ca=${ca.crt:output}&cert=${neo.crt:output}&key=${neo.key:output}' -%}
+{%   endif -%}
+[{{ section('reflink') }}]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:etc_run}/reflink
+{%   set args = [] -%}
+{%   if 'commit-interval' in reflink -%}
+{%     do args.extend(('-i', str(reflink['commit-interval']))) -%}
+{%   endif -%}
+{%   if 'period' in reflink -%}
+{%     do args.extend(('-p', str(reflink['period']))) -%}
+{%   endif -%}
+{%   if 'no-gc' in reflink -%}
+{%     do args.append('--no-gc') -%}
+{%   endif -%}
+command-line = {{ bin_directory }}/reflink -v {{ zurl }} run {{ ' '.join(args) }} {{ reflink['zurl'] }}
+{%   if reflink['zurl'].startswith('neos://') -%}
+environment =
+  NEO_CA=${reflink-ca:output}
+  NEO_CERT=${reflink-cert:output}
+  NEO_KEY=${reflink-key:output}
+
+{%     for x in 'ca', 'cert', 'key' -%}
+[reflink-{{x}}]
+recipe = slapos.recipe.template:jinja2
+output = ${directory:etc}/${:_buildout_section_name_}.pem
+inline =
+  {{'{{'}}pem}}
+context = key pem :pem
+pem = {{dumps(reflink['_'+x])}}
+{%     endfor -%}
+{%   endif -%}
+{% endif -%}
+
 {% if mysql -%}
 [init-script]
 recipe = slapos.recipe.template

software/neoppod/root-common.cfg.in

@@ -133,6 +133,9 @@ config-monitor-port = {{ dumps(port) }}
 {%-     endif %}
 config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }}
 {%-   endif %}
+{%- if 'reflink' in parameter_dict %}
+config-reflink = {{ dumps(parameter_dict.pop('reflink')) }}
+{%- endif %}
 {%-   for k, v in six.iteritems(node) %}
 config-{{ k }} = {{ dumps(v) }}
 {%-   endfor %}

stack/erp5-zope2/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 6d9a926e07b674ffdaecd381d763c068
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 6b10ab0c54278156caf058ebb7246645
+md5sum = 7ac187ff9fe306297724a730459e1a4d
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5-zope2/instance-erp5.cfg.in

@@ -186,6 +186,7 @@ connection-url = smtp://127.0.0.2:0/
 {%     set ((name, server_dict),) = server_dict.items() -%}
 {%     do neo.append(server_dict.get('cluster')) -%}
 {%     do server_dict.update(cluster='${publish-early:neo-cluster}') -%}
+{{     assert('reflink' not in server_dict, 'reflink option is meaningless in ERP5 SR') }}
 {{     root_common.request_neo(server_dict, 'zodb-neo', 'neo-', monitor_base_url_dict) }}
 {%     set client_dict = zodb_dict[name].setdefault('storage-dict', {}) -%}
 {%     for k in 'ssl', '_ca', '_cert', '_key' -%}

stack/erp5/buildout.cfg

@@ -529,6 +529,7 @@ eggs =
   lock_file
   astor
   APacheDEX
+  ${backports.lzma:egg}
   chardet
   collective.recipe.template
   erp5diff

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 8e452bd32fc0d4d858b275a2b3ee790b
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 1333d2fc21f64da4010a4eafea59d141
+md5sum = bd8cb060b37d8dc6a1ab4e25a2a7e109
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -94,7 +94,7 @@ md5sum = 9c580be982d8c63ec06fc273ef3cb971
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 409a7505548576ebf0e4d5cc218e0753
+md5sum = e9aa89754085bdc7a6fb9e53c0c97f9d
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in