Harden pre-production deployment with external environment and preflights

2026-09-16 00:22:19 +02:00
parent 5d3ae10036
commit 2813795c9d
17 changed files with 1002 additions and 76 deletions
@@ -0,0 +1,174 @@
+"""Run the real shell/helper in a disposable checkout with fake Git/Docker commands."""
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import preprod_config
+
+
+class DeploymentOrderingTests(unittest.TestCase):
+    def setUp(self):
+        temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(temporary.cleanup)
+        self.root = Path(temporary.name)
+        self.repo = self.root / 'pingu-concerts'
+        (self.repo / 'scripts').mkdir(parents=True)
+        self.env = self.root / 'preprod.env'
+        self.secret = self.root / 'firebase.json'
+        self.secret.write_text('synthetic test bytes, not a credential')
+        self.secret.chmod(0o600)
+        self.values = {name: 'synthetic-private-value' for name in preprod_config.REQUIRED}
+        self.values.update(PUSH_ENABLED='true', COOKIE_SECURE='true', FIREBASE_PROJECT_ID='metalcircle-30d9b',
+                           FIREBASE_SERVICE_ACCOUNT_FILE=str(self.secret))
+        self.write_env()
+        source = Path(__file__).resolve().parents[1]
+        script = (source / 'deploy-preprod.sh').read_text().replace(
+            "ENV_FILE='/home/kai/.config/metalcircle/preprod.env'", 'ENV_FILE=' + repr(str(self.env)))
+        (self.repo / 'scripts/deploy-preprod.sh').write_text(script)
+        shutil.copy(source / 'preprod_config.py', self.repo / 'scripts')
+        self.control = self.root / 'control.json'
+        self.control.write_text('{}')
+        self.trace = self.root / 'trace.jsonl'
+        self.bin = self.root / 'bin'
+        self.bin.mkdir()
+        shared = (f'CONTROL = {str(self.control)!r}\nTRACE = {str(self.trace)!r}\n'
+                  'import json, sys\nfrom pathlib import Path\n'
+                  'control = json.loads(Path(CONTROL).read_text())\n'
+                  'args = sys.argv[1:]\n'
+                  'with open(TRACE, "a") as trace: trace.write(json.dumps([Path(sys.argv[0]).name, *args]) + "\\n")\n')
+        self.executable('git', shared + f'''
+if args == ['rev-parse', '--show-toplevel']: print({str(self.repo)!r})
+elif args == ['branch', '--show-current']: print(control.get('branch', 'main'))
+elif args[0] == 'status':
+    if control.get('fail') == 'status': sys.exit(1)
+    print(control.get('dirty', ''))
+elif args[0] == 'log': print('abc123 synthetic test commit')
+elif args[0] == 'pull' and control.get('fail') == 'pull': sys.exit(1)
+''')
+        self.executable('sudo', shared + '''
+assert args[:3] == ['docker', 'compose', '--env-file']
+env_file = Path(args[3])
+assert args[4:8] == ['-f', 'compose.yml', '-f', 'compose.preprod.yml']
+action = args[8:]
+step = action[0]
+if step in ('run', 'exec'):
+    step += '-firebase' if 'push_preflight.py' in action else '-gitea'
+if control.get('no_pass') == step:
+    print('synthetic-private-value')
+    sys.exit(0)
+if control.get('fail') == step:
+    print('FAIL: gitea_http_403' if 'gitea' in step else 'FAIL: credential_service_account_mismatch')
+    print('synthetic-private-value', file=sys.stderr)
+    sys.exit(1)
+if action == ['config', '--no-interpolate', '--environment']:
+    print(env_file.read_text())
+elif action[0] in ('run', 'exec'):
+    print('PASS: synthetic preflight')
+    if control.get('edit') and step == 'run-gitea':
+        env_file.write_text(env_file.read_text() + '\\nCHANGED=true\\n')
+elif action[0] == 'logs':
+    print('private synthetic-private-value')
+    print('web | INFO \\"POST /private/synthetic-private-value HTTP/1.1\\" 200 OK')
+elif action[0] == 'ps': print(json.dumps([{'Name': 'pingu-concerts-web', 'State': 'running'}]))
+''')
+
+    def executable(self, name, code):
+        path = self.bin / name
+        path.write_text('#!/usr/bin/env python3\n' + code)
+        path.chmod(0o700)
+
+    def write_env(self):
+        self.env.write_text('\n'.join(k + '=' + v for k, v in self.values.items()))
+        self.env.chmod(0o600)
+
+    def run_deploy(self, *args, **controls):
+        self.control.write_text(json.dumps(controls))
+        self.trace.write_text('')
+        result = subprocess.run(['bash', str(self.repo / 'scripts/deploy-preprod.sh'), *args],
+                                cwd=self.root, env={**os.environ, 'PATH': str(self.bin) + ':' + os.environ['PATH']},
+                                capture_output=True, text=True)
+        self.assertNotIn('synthetic-private-value', result.stdout + result.stderr)
+        commands = [json.loads(line) for line in self.trace.read_text().splitlines()]
+        self.actions = [cmd[9:] for cmd in commands if cmd[0] == 'sudo']
+        return result
+
+    def assert_no_update(self, result):
+        self.assertNotEqual(result.returncode, 0)
+        self.assertFalse(any(action[0] == 'up' for action in self.actions))
+
+    def test_success_order_and_non_destructive_commands(self):
+        result = self.run_deploy()
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+        update = self.actions.index(['up', '-d', '--no-deps', 'web'])
+        preflights = [a for a in self.actions[:update] if a[0] == 'run']
+        self.assertEqual(len(preflights), 2)
+        self.assertTrue(all('--no-deps' in a and '--rm' in a for a in preflights))
+        self.assertIn('push_preflight.py', preflights[0])
+        self.assertIn('gitea_preflight.py', preflights[1])
+        self.assertEqual(len([a for a in self.actions[update + 1:] if a[0] == 'exec']), 2)
+        self.assertEqual(len(list((self.root / 'backups').glob('preprod.env.*'))), 1)
+        self.assertTrue(all(not set(a) & {'down', 'rm', '--force-recreate', 'db'} for a in self.actions))
+
+    def test_missing_env_secret_or_any_gitea_value_never_updates(self):
+        for missing in ('GITEA_URL', 'GITEA_TOKEN', 'GITEA_OWNER', 'GITEA_REPO'):
+            with self.subTest(missing=missing):
+                saved = self.values.pop(missing)
+                self.write_env()
+                result = self.run_deploy()
+                self.assert_no_update(result)
+                self.assertIn('ERROR: required variable ' + missing + ' is missing', result.stdout)
+                self.assertIn('Running container unchanged.', result.stderr)
+                self.values[missing] = saved
+        self.write_env()
+        self.env.unlink()
+        self.assert_no_update(self.run_deploy())
+        self.write_env()
+        self.secret.unlink()
+        self.assert_no_update(self.run_deploy())
+
+    def test_every_failure_before_up_keeps_running_container(self):
+        for fail in ('status', 'pull', 'config', 'build', 'run-firebase', 'run-gitea'):
+            with self.subTest(fail=fail):
+                result = self.run_deploy(fail=fail)
+                self.assert_no_update(result)
+                self.assertIn('Running container unchanged.', result.stderr)
+        self.assertFalse((self.root / 'backups').exists())
+
+    def test_dirty_tree_and_wrong_branch_abort_without_compose(self):
+        for controls in ({'branch': 'feature'}, {'dirty': ' M changed-file'}):
+            result = self.run_deploy(**controls)
+            self.assert_no_update(result)
+            self.assertEqual(self.actions, [])
+
+    def test_check_only_never_updates_or_backs_up(self):
+        result = self.run_deploy('--check')
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+        self.assertFalse(any(a[0] in ('up', 'exec') for a in self.actions))
+        self.assertFalse((self.root / 'backups').exists())
+
+    def test_config_edit_during_preflight_aborts(self):
+        result = self.run_deploy(edit=True)
+        self.assert_no_update(result)
+        self.assertIn('changed during deployment', result.stderr)
+
+    def test_zero_exit_without_pass_is_not_sufficient(self):
+        for step in ('run-firebase', 'run-gitea'):
+            with self.subTest(step=step):
+                self.assert_no_update(self.run_deploy(no_pass=step))
+
+    def test_backup_failure_aborts_before_update(self):
+        (self.root / 'backups').mkdir(mode=0o755)
+        self.assert_no_update(self.run_deploy())
+
+    def test_post_update_failure_does_not_claim_old_container_unchanged(self):
+        result = self.run_deploy(fail='exec-gitea')
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn('Manual investigation required.', result.stderr)
+        self.assertNotIn('Running container unchanged.', result.stderr)
+        self.assertEqual(len([a for a in self.actions if a[0] == 'up']), 1)
@@ -0,0 +1,173 @@
+from contextlib import redirect_stdout
+from io import StringIO
+import os
+from pathlib import Path
+import shutil
+import stat
+import subprocess
+import sys
+import tempfile
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import preprod_config as config
+
+
+class HostConfigurationTests(unittest.TestCase):
+    def setUp(self):
+        temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(temporary.cleanup)
+        self.root = Path(temporary.name)
+        self.repo = self.root / 'pingu-concerts'
+        self.repo.mkdir()
+        self.env = self.root / 'preprod.env'
+        self.env.write_text('synthetic fixture only')
+        self.env.chmod(0o600)
+        self.secret = self.root / 'firebase.json'
+        self.secret.write_text('synthetic fixture only; not credentials')
+        self.secret.chmod(0o600)
+        self.values = {name: 'test-value' for name in config.REQUIRED}
+        self.values.update(PUSH_ENABLED='true', COOKIE_SECURE='true', FIREBASE_PROJECT_ID='metalcircle-30d9b',
+                           FIREBASE_SERVICE_ACCOUNT_FILE=str(self.secret),
+                           COMPOSE_PROJECT_NAME='pingu-concerts', DOCKER_CLI_PLUGIN_ORIGINAL_CLI_COMMAND='docker compose')
+
+    def check(self):
+        response = SimpleNamespace(returncode=0, stdout='\n'.join(k + '=' + v for k, v in self.values.items()))
+        with patch.object(config, 'invoke_compose', return_value=response):
+            return config.check_configuration(self.env, self.repo)
+
+    def test_valid_configuration_and_auto_compose_metadata(self):
+        self.assertEqual(self.check()['PUSH_ENABLED'], 'true')
+
+    def test_each_missing_or_blank_required_variable_is_named_without_values(self):
+        for name in config.REQUIRED:
+            for value in ('', '   '):
+                with self.subTest(name=name, value=value), patch.dict(self.values, {name: value}):
+                    with self.assertRaisesRegex(config.PreprodError, '^required variable ' + name + ' is missing$'):
+                        self.check()
+
+    def test_missing_env_and_missing_firebase_secret(self):
+        for path, expected in ((self.env, 'Environment file'), (self.secret, 'Firebase secret')):
+            saved = path.read_bytes()
+            path.unlink()
+            with self.assertRaisesRegex(config.PreprodError, expected + ' is missing'):
+                self.check()
+            path.write_bytes(saved)
+            path.chmod(0o600)
+
+    def test_permissions_symlinks_and_repository_paths_are_rejected(self):
+        for path in (self.env, self.secret):
+            path.chmod(0o644)
+            with self.assertRaisesRegex(config.PreprodError, 'requires permissions'):
+                self.check()
+            path.chmod(0o600)
+        inside = self.repo / 'private.json'
+        inside.write_text('fixture')
+        inside.chmod(0o600)
+        self.values['FIREBASE_SERVICE_ACCOUNT_FILE'] = str(inside)
+        with self.assertRaisesRegex(config.PreprodError, 'outside the repository'):
+            self.check()
+        linked = self.root / 'linked.json'
+        linked.symlink_to(self.secret)
+        self.values['FIREBASE_SERVICE_ACCOUNT_FILE'] = str(linked)
+        with self.assertRaisesRegex(config.PreprodError, 'not a symlink'):
+            self.check()
+
+    def test_wrong_owner_and_directory_fail(self):
+        self.values['FIREBASE_SERVICE_ACCOUNT_FILE'] = str(self.root)
+        with self.assertRaisesRegex(config.PreprodError, 'regular file'):
+            self.check()
+        with patch('preprod_config.os.getuid', return_value=os.getuid() + 1):
+            with self.assertRaisesRegex(config.PreprodError, 'deployment user'):
+                config.private_file(self.env, self.repo, 'Environment file')
+
+    def test_disabled_push_insecure_cookies_wrong_project_and_context_overrides_fail(self):
+        for name, value in (('PUSH_ENABLED', 'false'), ('COOKIE_SECURE', 'false'),
+                            ('FIREBASE_PROJECT_ID', 'other'), ('COMPOSE_PROJECT_NAME', 'other'),
+                            ('DOCKER_HOST', 'tcp://elsewhere'), ('COMPOSE_ENV_FILES', '.env')):
+            with self.subTest(name=name), patch.dict(self.values, {name: value}):
+                with self.assertRaises(config.PreprodError):
+                    self.check()
+
+    def test_compose_parse_errors_never_expose_stderr(self):
+        response = SimpleNamespace(returncode=1, stderr='synthetic-secret', stdout='synthetic-secret')
+        with patch.object(config, 'invoke_compose', return_value=response):
+            with self.assertRaises(config.PreprodError) as caught:
+                config.check_configuration(self.env, self.repo)
+        self.assertNotIn('synthetic-secret', str(caught.exception))
+
+    def test_backups_are_private_keep_ten_and_preserve_unrelated_files(self):
+        directory = self.env.parent / 'backups'
+        directory.mkdir(mode=0o700)
+        unrelated = directory / 'keep-me'
+        unrelated.write_text('unrelated')
+        with redirect_stdout(StringIO()):
+            for _ in range(12):
+                config.backup_environment(self.env, self.repo)
+        backups = list(directory.glob('preprod.env.*'))
+        self.assertEqual(len(backups), 10)
+        self.assertEqual(stat.S_IMODE(directory.stat().st_mode), 0o700)
+        self.assertTrue(unrelated.exists())
+        for backup in backups:
+            self.assertEqual(stat.S_IMODE(backup.stat().st_mode), 0o600)
+            self.assertEqual(backup.read_bytes(), self.env.read_bytes())
+
+    def test_backup_refuses_insecure_directory(self):
+        directory = self.env.parent / 'backups'
+        directory.mkdir(mode=0o755)
+        with self.assertRaisesRegex(config.PreprodError, 'permissions 700'):
+            config.backup_environment(self.env, self.repo)
+
+    def test_compose_cannot_inherit_shell_secrets_or_remote_docker_context(self):
+        with patch.dict(os.environ, {'GITEA_TOKEN': 'stale', 'DOCKER_HOST': 'remote', 'COMPOSE_FILE': 'other'}):
+            cleaned = config.compose_environment()
+        for name in ('GITEA_TOKEN', 'DOCKER_HOST', 'COMPOSE_FILE'):
+            self.assertNotIn(name, cleaned)
+
+    def test_safe_logs_discard_private_text_and_request_urls(self):
+        output = StringIO()
+        with redirect_stdout(output):
+            config.safe_logs('web | INFO "POST /reset/synthetic-secret HTTP/1.1" 200 OK\n'
+                             'web | private synthetic-secret\n'
+                             'web | Gitea issue submission failed: HTTP 403\n'
+                             'web | Push delivery failed: configuration\n')
+        self.assertNotIn('synthetic-secret', output.getvalue())
+        self.assertIn('HTTP POST 200', output.getvalue())
+        self.assertIn('Gitea: HTTP 403', output.getvalue())
+        self.assertIn('Push: configuration', output.getvalue())
+
+    @unittest.skipUnless(shutil.which('docker'), 'requires Compose CLI only; no daemon/network')
+    def test_real_compose_config_requires_gitea_even_without_host_helper(self):
+        def validate():
+            self.env.write_text('\n'.join(k + '=' + v for k, v in self.values.items()
+                                          if not k.startswith(('COMPOSE_', 'DOCKER_'))))
+            return subprocess.run(['docker', 'compose', '--env-file', str(self.env),
+                                   '-f', str(config.REPO_ROOT / 'compose.yml'),
+                                   '-f', str(config.REPO_ROOT / 'compose.preprod.yml'), 'config', '--quiet'],
+                                  env=config.compose_environment(), capture_output=True, text=True)
+        self.assertEqual(validate().returncode, 0)
+        for name in ('GITEA_URL', 'GITEA_TOKEN', 'GITEA_OWNER', 'GITEA_REPO'):
+            with self.subTest(name=name), patch.dict(self.values, {name: ''}):
+                result = validate()
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn('required variable ' + name + ' is missing', result.stderr)
+
+    @unittest.skipUnless(shutil.which('docker'), 'requires Compose CLI only; no daemon/network')
+    def test_real_compose_dotenv_quoting_and_no_checkout_or_shell_fallback(self):
+        self.values['GITEA_TOKEN'] = "'synthetic-$literal#value'"
+        self.env.write_text('\n'.join(k + '=' + v for k, v in self.values.items()
+                                      if not k.startswith(('COMPOSE_', 'DOCKER_'))))
+        def invoke(env_file, args, repo_root):
+            return subprocess.run(['docker', 'compose', '--env-file', str(env_file),
+                                   '-f', str(config.REPO_ROOT / 'compose.yml'),
+                                   '-f', str(config.REPO_ROOT / 'compose.preprod.yml'), *args],
+                                  env=config.compose_environment(), capture_output=True, text=True)
+        with patch.object(config, 'invoke_compose', side_effect=invoke):
+            checked = config.check_configuration(self.env, config.REPO_ROOT)
+            self.assertEqual(checked['GITEA_TOKEN'], 'synthetic-$literal#value')
+            self.env.write_text(self.env.read_text().replace("GITEA_TOKEN='synthetic-$literal#value'", ''))
+            with patch.dict(os.environ, {'GITEA_TOKEN': 'must-not-be-used'}):
+                with self.assertRaisesRegex(config.PreprodError, 'required variable GITEA_TOKEN is missing'):
+                    config.check_configuration(self.env, config.REPO_ROOT)