219 lines
13 KiB
Python
Executable File
219 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Update reviewed sibling Git commits, build, and optionally activate a quiet deployment."""
|
|
import argparse
|
|
from contextlib import contextmanager
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
|
|
def run(args, cwd=None, capture=True):
|
|
result = subprocess.run(args, cwd=cwd, check=True, text=True,
|
|
stdout=subprocess.PIPE if capture else None)
|
|
return result.stdout.strip() if capture else None
|
|
|
|
|
|
@contextmanager
|
|
def staged_repositories(repositories):
|
|
with tempfile.TemporaryDirectory(prefix='otche-update-', dir='/var/tmp') as temporary:
|
|
root = Path(temporary)
|
|
added = []
|
|
try:
|
|
for directory, revision in repositories:
|
|
target = root / directory.name
|
|
run(['git', 'worktree', 'add', '--detach', str(target), revision], directory, False)
|
|
added.append((directory, target))
|
|
yield root
|
|
finally:
|
|
failures = []
|
|
for directory, target in reversed(added):
|
|
for command in (['git', 'worktree', 'remove', '--force', str(target)],
|
|
['git', 'worktree', 'prune']):
|
|
if subprocess.run(command, cwd=directory, check=False).returncode:
|
|
failures.append(str(target))
|
|
if failures:
|
|
raise RuntimeError('Temporary worktree cleanup failed: ' + ', '.join(failures))
|
|
|
|
|
|
def check_volumes(parser, compose, config, project, running):
|
|
if config['name'] != project:
|
|
parser.error('Resolved Compose project identity changed')
|
|
for logical in ('postgres-data', 'artifacts'):
|
|
volume = config['volumes'][logical]
|
|
if not volume.get('external') or not volume.get('name'):
|
|
parser.error('Existing deployment requires explicit external ' + logical + ' volume')
|
|
run(['docker', 'volume', 'inspect', volume['name']])
|
|
if running:
|
|
for service, destination, logical in (('postgres', '/var/lib/postgresql/data', 'postgres-data'),
|
|
('api', '/var/lib/otche', 'artifacts')):
|
|
containers = run(compose + ['ps', '-a', '-q', service]).splitlines()
|
|
if len(containers) != 1:
|
|
parser.error('Expected exactly one existing ' + service + ' container; inspect deployment before updating')
|
|
actual = json.loads(run(['docker', 'inspect', containers[0]]))[0]
|
|
if service == 'postgres' and not actual['State']['Running']:
|
|
parser.error('Existing database must be running')
|
|
if service == 'api' and actual['State']['Status'] not in ('running', 'exited'):
|
|
parser.error('API must be running or intentionally fenced (exited)')
|
|
mounts = actual['Mounts']
|
|
if not any(m.get('Name') == config['volumes'][logical]['name'] and m['Destination'] == destination for m in mounts):
|
|
parser.error('Persistent volume identity would change for ' + service)
|
|
|
|
|
|
def image_name(config, service):
|
|
return config['services'][service].get('image') or config['name'] + '-' + service
|
|
|
|
|
|
def verify_images(parser, compose, images):
|
|
for service, expected in images.items():
|
|
containers = run(compose + ['ps', '-a', '-q', service]).splitlines()
|
|
if not containers:
|
|
parser.error('Missing updated service container: ' + service)
|
|
for container in containers:
|
|
actual = json.loads(run(['docker', 'inspect', container]))[0]
|
|
if actual['Image'] != expected:
|
|
parser.error('Container image mismatch for ' + service + '; approved pins NOT changed')
|
|
state = actual['State']
|
|
if service == 'migrate':
|
|
ready = state['Status'] == 'exited' and state['ExitCode'] == 0
|
|
else:
|
|
ready = state['Running'] and state.get('Health', {}).get('Status', 'healthy') == 'healthy'
|
|
if not ready:
|
|
parser.error('Updated service is not ready: ' + service + '; approved pins NOT changed')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
for repo in ('backend', 'frontend', 'deploy'):
|
|
parser.add_argument('--' + repo, required=True, help='Reviewed full Git commit SHA')
|
|
parser.add_argument('--env-file', type=Path, required=True, help='Protected external environment file')
|
|
parser.add_argument('--override', type=Path, required=True, help='Protected external Compose override')
|
|
parser.add_argument('--project', required=True, help='EXISTING Compose project name; never rename it')
|
|
parser.add_argument('--activate', action='store_true')
|
|
parser.add_argument('--verify-only', action='store_true', help='Verify exact clean local pins and existing volumes without network/build/activation')
|
|
parser.add_argument('--confirm-quiescent', action='store_true', help='Operator has paused new submissions and coordinated execution owners')
|
|
args = parser.parse_args()
|
|
if os.geteuid() != 0:
|
|
parser.error('Run in the root administration shell so private bind sources remain protected')
|
|
if args.activate and not args.confirm_quiescent:
|
|
parser.error('Activation requires explicit coordinated quiescence')
|
|
if args.verify_only and args.activate:
|
|
parser.error('Verification and activation are separate modes')
|
|
if not re.fullmatch(r'[a-z0-9][a-z0-9_-]*', args.project):
|
|
parser.error('Invalid Compose project name')
|
|
deploy = Path(__file__).resolve().parent
|
|
workspace = deploy.parent
|
|
for private in (args.env_file, args.override):
|
|
if not private.is_absolute() or not private.is_file() or workspace in private.resolve().parents:
|
|
parser.error('Environment and override must be existing absolute files OUTSIDE the Git workspace')
|
|
repositories = []
|
|
for component in ('backend', 'frontend', 'deploy'):
|
|
directory = workspace / ('otche-' + component)
|
|
revision = getattr(args, component)
|
|
if not re.fullmatch(r'[0-9a-f]{40}', revision):
|
|
parser.error('Every revision must be a full reviewed SHA-1 commit')
|
|
if run(['git', 'status', '--porcelain', '--untracked-files=all'], directory):
|
|
parser.error(str(directory) + ' is dirty; preserve/review work, do not reset it')
|
|
expected = 'https://git.qomar.pw/otche/otche-' + component + '.git'
|
|
if run(['git', 'remote', 'get-url', 'origin'], directory) != expected:
|
|
parser.error(str(directory) + ' has an unexpected origin')
|
|
if run(['git', 'branch', '--show-current'], directory) != 'main':
|
|
parser.error(str(directory) + ' must be on main, not a detached or private branch')
|
|
if args.verify_only:
|
|
if run(['git', 'rev-parse', 'HEAD'], directory) != revision:
|
|
parser.error(str(directory) + ' does not match its approved startup pin')
|
|
else:
|
|
run(['git', '-c', 'credential.helper=', 'fetch', 'origin', 'main'], directory, False)
|
|
run(['git', 'cat-file', '-e', revision + '^{commit}'], directory)
|
|
run(['git', 'merge-base', '--is-ancestor', 'HEAD', revision], directory)
|
|
run(['git', 'merge-base', '--is-ancestor', revision, 'origin/main'], directory)
|
|
repositories.append((directory, revision))
|
|
compose = ['docker', 'compose', '--env-file', str(args.env_file), '-p', args.project,
|
|
'-f', str(deploy / 'compose.yaml'), '-f', str(args.override), '--profile', 'execution']
|
|
config = json.loads(run(compose + ['config', '--format', 'json']))
|
|
check_volumes(parser, compose, config, args.project, not args.verify_only)
|
|
if args.verify_only:
|
|
print('Exact clean repository pins and external persistent volumes verified')
|
|
return
|
|
# Build reviewed targets without moving checkouts or replacing runtime image tags.
|
|
with staged_repositories(repositories) as staging:
|
|
staged_compose = ['docker', 'compose', '--env-file', str(args.env_file), '-p', args.project,
|
|
'-f', str(staging / 'otche-deploy' / 'compose.yaml'), '-f', str(args.override),
|
|
'--profile', 'execution']
|
|
target_config = json.loads(run(staged_compose + ['config', '--format', 'json']))
|
|
check_volumes(parser, compose, target_config, args.project, True)
|
|
build_key = hashlib.sha256((args.backend + args.frontend + args.deploy).encode()).hexdigest()
|
|
built = {service: {'image': 'otche-update-' + args.project + '-' + service + ':' + build_key}
|
|
for service, settings in target_config['services'].items() if 'build' in settings}
|
|
for service, settings in built.items():
|
|
if service not in ('api', 'migrate', 'worker', 'watchdog', 'web'):
|
|
parser.error('Unknown build service requires explicit reviewed source mapping: ' + service)
|
|
source = 'otche-frontend' if service == 'web' else 'otche-backend'
|
|
settings['build'] = {'context': str(staging / source)}
|
|
build_override = staging / 'build-images.json'
|
|
build_override.write_text(json.dumps({'services': built}))
|
|
staged_compose += ['-f', str(build_override)]
|
|
run(staged_compose + ['build'], capture=False)
|
|
images = {service: run(['docker', 'image', 'inspect', '--format', '{{.Id}}', settings['image']])
|
|
for service, settings in built.items()}
|
|
if args.activate:
|
|
old_images = {}
|
|
for service in images:
|
|
containers = run(compose + ['ps', '-a', '-q', service]).splitlines()
|
|
old_images[service] = [json.loads(run(['docker', 'inspect', container]))[0]['Image']
|
|
for container in containers]
|
|
# Stop only API admission, after build; workers keep their existing containers.
|
|
old_api = run(compose + ['ps', '-a', '-q', 'api'])
|
|
api_was_running = json.loads(run(['docker', 'inspect', old_api]))[0]['State']['Running']
|
|
if api_was_running:
|
|
run(['docker', 'stop', '--time', '30', old_api], capture=False)
|
|
query = "SELECT (SELECT count(*) FROM jobs WHERE status IN ('queued','running')) + (SELECT count(*) FROM qualifications WHERE status IN ('queued','running'));"
|
|
try:
|
|
active = run(compose + ['exec', '-T', 'postgres', 'psql', '-U', 'otche', '-d', 'otche', '-At', '-c', query])
|
|
if active != '0':
|
|
parser.error('Active work remains; refusing activation and restarting the SAME old API container')
|
|
for directory, revision in repositories:
|
|
if run(['git', 'status', '--porcelain', '--untracked-files=all'], directory):
|
|
parser.error(str(directory) + ' changed during build; refusing activation')
|
|
run(['git', 'merge-base', '--is-ancestor', 'HEAD', revision], directory)
|
|
except BaseException:
|
|
if api_was_running:
|
|
run(['docker', 'start', old_api], capture=False)
|
|
raise
|
|
try:
|
|
# Only the successful fence permits advancing source and runtime image tags.
|
|
for directory, revision in repositories:
|
|
run(['git', 'merge', '--ff-only', revision], directory, False)
|
|
for service, image in images.items():
|
|
run(['docker', 'image', 'tag', image, image_name(target_config, service)], capture=False)
|
|
run(compose + ['up', '-d', '--no-build', '--force-recreate', '--wait', '--wait-timeout', '120'] + sorted(images), capture=False)
|
|
# Recreate web explicitly even when its image is unchanged, then await health.
|
|
run(compose + ['up', '-d', '--no-deps', '--no-build', '--force-recreate', '--wait', '--wait-timeout', '120', 'web'], capture=False)
|
|
verify_images(parser, compose, images)
|
|
pin_file = args.env_file.parent / 'approved-commits.env'
|
|
temporary = pin_file.with_suffix('.env.new')
|
|
with temporary.open('x') as output:
|
|
os.chmod(temporary, 0o600)
|
|
for component in ('backend', 'frontend', 'deploy'):
|
|
output.write('OTCHE_' + component.upper() + '_COMMIT=' + getattr(args, component) + '\n')
|
|
os.replace(temporary, pin_file)
|
|
except BaseException:
|
|
print('Activation failed; previous container image IDs for manual recovery (not deleted):', file=sys.stderr)
|
|
print(json.dumps(old_images, indent=2), file=sys.stderr)
|
|
raise
|
|
print(json.dumps({'activated': args.activate, 'project': args.project,
|
|
'commits': {d.name: r for d, r in repositories}}, indent=2))
|
|
if args.activate:
|
|
print('Verify authenticated health, prior data/evidence continuity and actual acceptance before resuming submissions.')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except subprocess.CalledProcessError as error:
|
|
sys.exit('Update stopped on command failure (exit %s); inspect state without reset/force.' % error.returncode)
|