From a1ae079e8a191d8e78dacf10b4c861c8335f30a0 Mon Sep 17 00:00:00 2001 From: redxef Date: Sat, 18 Feb 2023 20:48:22 +0100 Subject: [PATCH] Switch config validation. Allow multiple sources/sinks. --- adapters/__init__.py | 25 ++++- adapters/abc.py | 42 +++++++ adapters/google.py | 33 ++++-- adapters/wordpress.py | 42 ++++--- main.py | 250 ++++++++++++++++-------------------------- requirements.txt | 1 + 6 files changed, 214 insertions(+), 179 deletions(-) create mode 100644 adapters/abc.py diff --git a/adapters/__init__.py b/adapters/__init__.py index 9b2a713..731d2a4 100644 --- a/adapters/__init__.py +++ b/adapters/__init__.py @@ -1,11 +1,28 @@ - - from .google import Google from .wordpress import Wordpress -from .wordpress import CalendarMetadata +from .abc import Source, Sink, Adapter + +_ADAPTERS: set[type[Adapter]] = { + Google, + Wordpress, +} +_SOURCES: set[type[Source]] = { + Google, +} +_SINKS: set[type[Sink]] = { + Wordpress, +} +ADAPTERS: dict[str, type[Adapter]] = { + str(x.__name__).lower(): x for x in _ADAPTERS +} +SOURCES: dict[str, type[Source]] = { + str(x.__name__).lower(): x for x in _SOURCES +} +SINKS: dict[str, type[Sink]] = { + str(x.__name__).lower(): x for x in _SINKS +} __all__ = [ 'Google', 'Wordpress', - 'CalendarMetadata', ] diff --git a/adapters/abc.py b/adapters/abc.py new file mode 100644 index 0000000..6607fda --- /dev/null +++ b/adapters/abc.py @@ -0,0 +1,42 @@ +import abc +import typing +import datetime + +import schema + +class Adapter(abc.ABC): + schema: schema.Schema + + def __init__(self, config, *args, **kwargs): + _ = args, kwargs + self.config = config + + @classmethod + def new(cls, config: dict, *args, **kwargs): + return cls(cls.schema.validate(config) , *args, **kwargs) + + @abc.abstractmethod + def login(self) -> bool: + raise NotImplementedError() + +class Source(abc.ABC): + + @abc.abstractmethod + def get_events( + self, + start: datetime.datetime | None=None, + until: datetime.timedelta | None=None, + limit=None, + ) -> typing.Iterable[dict]: + raise NotImplementedError() + +class Sink(abc.ABC): + + @abc.abstractmethod + def post_events( + self, + events, + start: datetime.datetime | None=None, + until: datetime.timedelta | None=None, + ) -> bool: + raise NotImplementedError() diff --git a/adapters/google.py b/adapters/google.py index fdd77a5..472d5fb 100644 --- a/adapters/google.py +++ b/adapters/google.py @@ -3,6 +3,7 @@ import datetime import logging +import os from google.auth.transport.requests import Request #from google.oauth2.credentials import Credentials @@ -10,27 +11,45 @@ from google.oauth2.service_account import Credentials from google_auth_oauthlib.flow import InstalledAppFlow from googleapiclient.discovery import build from googleapiclient.errors import HttpError +from schema import Use, Optional, Schema logger = logging.getLogger(f"wp_cal.{__name__}") -print(logger.name) + +from .abc import Source, Adapter # If modifying these scopes, delete the file token.json. SCOPES = ['https://www.googleapis.com/auth/calendar.readonly'] -class Google(): +class Google(Source, Adapter): - def __init__(self, calendar_id, *, credentials, token_file): - self.calendar_id = calendar_id - self.credentials_info = credentials + schema = Schema({ + 'calendar_id': Use(str), + 'credentials': Use(dict), + Optional( + 'token_file', + default=os.path.join( + os.environ["HOME"], + '.wp-cal-google-token', + ), + ): Use(str), + }) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.calendar_id = self.config['calendar_id'] + self.credentials_info = self.config['credentials'] + self.token_file = self.config['token_file'] + + # runtime data self.credentials = None - def login(self): + def login(self) -> bool: self.credentials = Credentials.from_service_account_info(self.credentials_info, scopes=SCOPES) if not self.credentials.valid: self.credentials.refresh(Request()) logger.debug('credentials.valid = %s', repr(self.credentials.valid)) - + return self.credentials.valid def get_events(self, start: datetime.datetime | None=None, until: datetime.timedelta | None=None, limit=None): until = until if until else datetime.timedelta(days=365) diff --git a/adapters/wordpress.py b/adapters/wordpress.py index 4081716..b9c2582 100644 --- a/adapters/wordpress.py +++ b/adapters/wordpress.py @@ -7,7 +7,10 @@ import json import datetime import logging +from schema import Use, Schema + from . import utils +from .abc import Sink, Adapter logger = logging.getLogger(f"wp_cal.{__name__}") @@ -33,19 +36,29 @@ class CalendarMetadata(): **translations, } -class Wordpress(): +class Wordpress(Sink, Adapter): - def __init__( - self, - base_url: str, - *, - calendar_metadata: CalendarMetadata, - credentials: dict, - ): - self.base_url = base_url + schema = Schema({ + 'url': Use(str), + 'calendar': { + 'id': Use(str), + 'name': Use(str), + 'translations': Use(dict), + }, + 'credentials': { + 'user': Use(str), + 'password': Use(str), + }, + }) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.base_url = self.config['url'] + self.credentials = self.config['credentials'] + self.calendar_metadata = CalendarMetadata(**self.config['calendar']) + + # runtime data self.session = requests.Session() - self.credentials = credentials - self.calendar_meadata = calendar_metadata def login(self): login_request = self.session.post( @@ -139,7 +152,7 @@ class Wordpress(): return final_dict def post_events(self, events, start: datetime.datetime | None=None, until: datetime.timedelta | None=None): - metadata = self.calendar_meadata.to_dict() + metadata = self.calendar_metadata.to_dict() data = self._generate_data(events, start=start, until=until) update_request = self.session.post( f'{self.base_url}/wp-admin/admin-ajax.php', @@ -150,5 +163,8 @@ class Wordpress(): 'calendar_data': json.dumps(data), }, ) - return 'wpbs_message=calendar_update_success' in update_request.text + r = 'wpbs_message=calendar_update_success' in update_request.text + if not r: + raise Exception('failed to post events') + diff --git a/main.py b/main.py index 60a2bda..8ea01dd 100755 --- a/main.py +++ b/main.py @@ -1,134 +1,21 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import functools import os import sys -import yaml import logging import datetime -try: - from yaml import CLoader as Loader, CDumper as Dumper -except ImportError: - from yaml import Loader, Dumper +import yaml +from schema import Schema, And, Or, Use, Optional, Regex import click +import adapters from adapters import * logger = logging.getLogger(f'wp_cal') -class BaseConfig(dict): - - def __init__(self, file, defaults, overrides, *args, **kwargs): - self._ex = None - self.file = file - self.defaults = defaults - self.overrides = overrides - super().__init__(*args, **kwargs) - try: - self._load() - except FileNotFoundError as ex: - self._ex = ex - - def __repr__(self): - return super().__repr__() - - def __str__(self): - return super().__str__() - - def _load(self): - try: - if self.file == '-': - config = yaml.safe_load(sys.stdin) - else: - if os.stat(self.file).st_mode & 0o777 & ~0o600: - raise Exception('refusing to load insecure configuration file, file must have permission 0o600') - with open(self.file) as fp: - config = yaml.safe_load(fp) - if config is None: - config = {} - for k, v in config.items(): - self[k] = v - finally: - for keylist, value in self.defaults: - d = self - for i, key in enumerate(keylist): - repl = value if (i == len(keylist) - 1) else {} - d[key] = d.get(key, repl) - d = d[key] - for keylist, value in self.overrides: - d = self - for key in keylist[:-1]: - d[key] = d.get(key, {}) - d = d[key] - d[keylist[-1]] = value - - def _save(self): - with open(self.file, 'w') as fp: - yaml.safe_dump(dict(self), fp) - - def exception(self): - return self._ex - - def load(self): - return self._load() - - def save(self): - return self._save() - -class Config(BaseConfig): - """ - The default configuration. - - - Keys: - .google.calendar_id: the id of the calendar to sync - .google.credentials: the json of the obtained credentials file - .google.token_file: where to save the login token - .wordpress.url: the base wordpress url - .wordpress.calendar.id: the id of the (wp-booking-system) calendar - .wordpress.calendar.name: the name of the calendar - .wordpress.calendar.translations: a dictionary of language <-> translation pairs (example: {"en": "Reservations"}) - .wordpress.credentials.user: the user as which to log into wordpress - .wordpress.credentials.password: the users password - .logging.level: one or more log levels of the form : seperated by a `,` (comma) - .range: the range in the format , where n are digits and u is the unit; valid units: 'smhdw' <-> seconds minutes hours days weeks - """ - - def __init__(self, file, defaults, overrides, converters, *args, **kwargs): - defaults += [ - ('google.calendar_id', '#TODO insert google calendar id'), - ('google.credentials', {}), - ('google.token_file', os.path.join(os.environ['HOME'], '.wp-cal-google-token')), - ('wordpress.url', '#TODO insert url to wordpress site'), - ('wordpress.calendar.id', '#TODO insert wp-booking-system calendar id'), - ('wordpress.calendar.name', '#TODO insert calendar name'), - ('wordpress.calendar.translations', {'en': '#TODO insert english translation'}), - ('wordpress.credentials.user', '#TODO insert wordpress username'), - ('wordpress.credentials.password', '#TODO insert wordpress password'), - ('logging.level', ':WARNING,wp_cal:INFO'), - ('range', '365d'), - ] - defaults = [(x[0].split('.'), x[1]) for x in defaults] - overrides = [(x[0].split('.'), x[1]) for x in overrides] - self.converters = converters - super().__init__(file, defaults, overrides, *args, **kwargs) - - def value(self, k): - if isinstance(k, str): - sel = k.split('.') - elif isinstance(k, list): - sel = k - k = '.'.join(k) - else: - raise TypeError('invalid type for parameter k') - v = self - for s in sel: - v = v[s] - if k in self.converters: - return self.converters[k](v) - return v - def range_str_to_timedelta(value): valid_units = 'smhdw' current_int = 0 @@ -157,12 +44,37 @@ def range_str_to_timedelta(value): weeks=values['w'] ) +config_schema = Schema({ + 'sources': [dict], + 'sinks': [dict], + Optional('logging', default={'level': ':WARNING,wp_cal:INFO'}): { + Optional('level', default=':WARNING,wp_cal:INFO'): Use(str), + }, + Optional('range', default='365d'): Or( + And( + Use(str), + Regex(r'[0-9]+[smhdw]'), + Use(range_str_to_timedelta), + ), + lambda x: isinstance(x, datetime.timedelta) + ), +}) + +def load_config(file): + if file == '-': + config = yaml.safe_load(sys.stdin) + else: + if os.stat(file).st_mode & 0o777 & ~0o600: + raise Exception('refusing to load insecure configuration file, file must have permission 0o600') + with open(file) as fp: + config = yaml.safe_load(fp) + return config_schema.validate(config) + def init_logging(): logging.getLogger().addHandler( logging.StreamHandler(), ) - def set_logging_level(level: str): levels = { 'NOTSET': logging.NOTSET, @@ -191,50 +103,78 @@ def main(config, dryrun, level, range): init_logging() set_logging_level(':DEBUG') - config_overrides = [('logging.level', level), ('range', range)] - config_overrides = [x for x in config_overrides if x[1] is not None] - converters = { - 'range': range_str_to_timedelta, - } - config = Config(config, [], config_overrides, converters) - if config.exception(): - logger.info('config not found, trying to generate template') - try: - config.save() - except Exception: - logger.exception('failed to generate template') + config = load_config(config) + if level: + config['logging']['level'] = level + if range: + config['range'] = range + config = config_schema.validate(config) + set_logging_level(config['logging']['level']) + + sources = set() + sinks = set() + for source in config['sources']: + assert len(source.keys()) == 1 + for aname, a in adapters.ADAPTERS.items(): + if aname in source: + sources |= {a.new(source[aname])} + break else: - logger.info('generated config at "%s"', config.file) - return + logger.error('couldn\'t find valid adapter for source configuration %s', source) + return 1 + for sink in config['sinks']: + assert len(sink.keys()) == 1 + for aname, a in adapters.ADAPTERS.items(): + if aname in sink: + sinks |= {a.new(sink[aname])} + break + else: + logger.error('couldn\'t find valid adapter for sink configuration %s', sink) + return 1 - set_logging_level(str(config.value('logging.level'))) - - g = Google( - config.value('google.calendar_id'), - credentials=config.value('google.credentials'), - token_file=config.value('google.token_file'), - ) - w = Wordpress( - config.value('wordpress.url'), - calendar_metadata=CalendarMetadata( - id=config.value('wordpress.calendar.id'), - name=config.value('wordpress.calendar.name'), - translations=config.value('wordpress.calendar.translations'), - ), - credentials=config.value('wordpress.credentials'), - ) - g.login() - events = g.get_events(until=config.value('range')) - logger.info("syncing %d events", len(events)) - if not w.login(): - logger.info('failed to login to wordpress') + if not all([isinstance(x, adapters.Source) for x in sources]): + logger.error('one or more source configurations do not implement being a source') return 1 + if not all([isinstance(x, adapters.Sink) for x in sinks]): + logger.error('one or more sink configurations do not implement being a sink') + return 1 + + # log in + if not all([x.login() for x in sources | sinks]): + logger.error('failed to log into one or more sinks or sources') + return 1 + + # gather events + events = [] + source_results = [] + for source in sources: + try: + events += source.get_events(until=config['range']) + source_results += [True] + except Exception: + logger.exception('failed to get events from source %s', source) + source_results += [False] + if not any(source_results): + logger.error('event get failed for all sources') + return 1 + logger.info("syncing %d events", len(events)) + + # post events if dryrun: logger.info("dryrun; would post events: %s", events) else: - if not w.post_events(events, until=config.value('range')): - logger.info('failed to post event data') + sink_results = [] + for sink in sinks: + try: + sink.post_events(events, until=config['range']) + sink_results += [True] + except Exception: + logger.exception('failed to post to sink %s', sink) + sink_results += [False] + if not any(sink_results): + logger.error('event post failed for all sinks') return 1 + logger.info("done") return 0 diff --git a/requirements.txt b/requirements.txt index 7ff00e0..b5d8814 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ pyyaml +schema google-api-python-client google-auth-httplib2 google-auth-oauthlib