Switch config validation. Allow multiple sources/sinks.

This commit is contained in:
redxef 2023-02-18 20:48:22 +01:00
parent d08fea10b8
commit a1ae079e8a
Signed by: redxef
GPG key ID: 7DAC3AA211CBD921
6 changed files with 214 additions and 179 deletions

View file

@ -1,11 +1,28 @@
from .google import Google from .google import Google
from .wordpress import Wordpress from .wordpress import Wordpress
from .wordpress import CalendarMetadata from .abc import Source, Sink, Adapter
_ADAPTERS: set[type[Adapter]] = {
Google,
Wordpress,
}
_SOURCES: set[type[Source]] = {
Google,
}
_SINKS: set[type[Sink]] = {
Wordpress,
}
ADAPTERS: dict[str, type[Adapter]] = {
str(x.__name__).lower(): x for x in _ADAPTERS
}
SOURCES: dict[str, type[Source]] = {
str(x.__name__).lower(): x for x in _SOURCES
}
SINKS: dict[str, type[Sink]] = {
str(x.__name__).lower(): x for x in _SINKS
}
__all__ = [ __all__ = [
'Google', 'Google',
'Wordpress', 'Wordpress',
'CalendarMetadata',
] ]

42
adapters/abc.py Normal file
View file

@ -0,0 +1,42 @@
import abc
import typing
import datetime
import schema
class Adapter(abc.ABC):
schema: schema.Schema
def __init__(self, config, *args, **kwargs):
_ = args, kwargs
self.config = config
@classmethod
def new(cls, config: dict, *args, **kwargs):
return cls(cls.schema.validate(config) , *args, **kwargs)
@abc.abstractmethod
def login(self) -> bool:
raise NotImplementedError()
class Source(abc.ABC):
@abc.abstractmethod
def get_events(
self,
start: datetime.datetime | None=None,
until: datetime.timedelta | None=None,
limit=None,
) -> typing.Iterable[dict]:
raise NotImplementedError()
class Sink(abc.ABC):
@abc.abstractmethod
def post_events(
self,
events,
start: datetime.datetime | None=None,
until: datetime.timedelta | None=None,
) -> bool:
raise NotImplementedError()

View file

@ -3,6 +3,7 @@
import datetime import datetime
import logging import logging
import os
from google.auth.transport.requests import Request from google.auth.transport.requests import Request
#from google.oauth2.credentials import Credentials #from google.oauth2.credentials import Credentials
@ -10,27 +11,45 @@ from google.oauth2.service_account import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build from googleapiclient.discovery import build
from googleapiclient.errors import HttpError from googleapiclient.errors import HttpError
from schema import Use, Optional, Schema
logger = logging.getLogger(f"wp_cal.{__name__}") logger = logging.getLogger(f"wp_cal.{__name__}")
print(logger.name)
from .abc import Source, Adapter
# If modifying these scopes, delete the file token.json. # If modifying these scopes, delete the file token.json.
SCOPES = ['https://www.googleapis.com/auth/calendar.readonly'] SCOPES = ['https://www.googleapis.com/auth/calendar.readonly']
class Google(): class Google(Source, Adapter):
def __init__(self, calendar_id, *, credentials, token_file): schema = Schema({
self.calendar_id = calendar_id 'calendar_id': Use(str),
self.credentials_info = credentials 'credentials': Use(dict),
Optional(
'token_file',
default=os.path.join(
os.environ["HOME"],
'.wp-cal-google-token',
),
): Use(str),
})
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.calendar_id = self.config['calendar_id']
self.credentials_info = self.config['credentials']
self.token_file = self.config['token_file']
# runtime data
self.credentials = None self.credentials = None
def login(self): def login(self) -> bool:
self.credentials = Credentials.from_service_account_info(self.credentials_info, scopes=SCOPES) self.credentials = Credentials.from_service_account_info(self.credentials_info, scopes=SCOPES)
if not self.credentials.valid: if not self.credentials.valid:
self.credentials.refresh(Request()) self.credentials.refresh(Request())
logger.debug('credentials.valid = %s', repr(self.credentials.valid)) logger.debug('credentials.valid = %s', repr(self.credentials.valid))
return self.credentials.valid
def get_events(self, start: datetime.datetime | None=None, until: datetime.timedelta | None=None, limit=None): def get_events(self, start: datetime.datetime | None=None, until: datetime.timedelta | None=None, limit=None):
until = until if until else datetime.timedelta(days=365) until = until if until else datetime.timedelta(days=365)

View file

@ -7,7 +7,10 @@ import json
import datetime import datetime
import logging import logging
from schema import Use, Schema
from . import utils from . import utils
from .abc import Sink, Adapter
logger = logging.getLogger(f"wp_cal.{__name__}") logger = logging.getLogger(f"wp_cal.{__name__}")
@ -33,19 +36,29 @@ class CalendarMetadata():
**translations, **translations,
} }
class Wordpress(): class Wordpress(Sink, Adapter):
def __init__( schema = Schema({
self, 'url': Use(str),
base_url: str, 'calendar': {
*, 'id': Use(str),
calendar_metadata: CalendarMetadata, 'name': Use(str),
credentials: dict, 'translations': Use(dict),
): },
self.base_url = base_url 'credentials': {
'user': Use(str),
'password': Use(str),
},
})
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.base_url = self.config['url']
self.credentials = self.config['credentials']
self.calendar_metadata = CalendarMetadata(**self.config['calendar'])
# runtime data
self.session = requests.Session() self.session = requests.Session()
self.credentials = credentials
self.calendar_meadata = calendar_metadata
def login(self): def login(self):
login_request = self.session.post( login_request = self.session.post(
@ -139,7 +152,7 @@ class Wordpress():
return final_dict return final_dict
def post_events(self, events, start: datetime.datetime | None=None, until: datetime.timedelta | None=None): def post_events(self, events, start: datetime.datetime | None=None, until: datetime.timedelta | None=None):
metadata = self.calendar_meadata.to_dict() metadata = self.calendar_metadata.to_dict()
data = self._generate_data(events, start=start, until=until) data = self._generate_data(events, start=start, until=until)
update_request = self.session.post( update_request = self.session.post(
f'{self.base_url}/wp-admin/admin-ajax.php', f'{self.base_url}/wp-admin/admin-ajax.php',
@ -150,5 +163,8 @@ class Wordpress():
'calendar_data': json.dumps(data), 'calendar_data': json.dumps(data),
}, },
) )
return 'wpbs_message=calendar_update_success' in update_request.text r = 'wpbs_message=calendar_update_success' in update_request.text
if not r:
raise Exception('failed to post events')

252
main.py
View file

@ -1,134 +1,21 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import functools
import os import os
import sys import sys
import yaml
import logging import logging
import datetime import datetime
try: import yaml
from yaml import CLoader as Loader, CDumper as Dumper from schema import Schema, And, Or, Use, Optional, Regex
except ImportError:
from yaml import Loader, Dumper
import click import click
import adapters
from adapters import * from adapters import *
logger = logging.getLogger(f'wp_cal') logger = logging.getLogger(f'wp_cal')
class BaseConfig(dict):
def __init__(self, file, defaults, overrides, *args, **kwargs):
self._ex = None
self.file = file
self.defaults = defaults
self.overrides = overrides
super().__init__(*args, **kwargs)
try:
self._load()
except FileNotFoundError as ex:
self._ex = ex
def __repr__(self):
return super().__repr__()
def __str__(self):
return super().__str__()
def _load(self):
try:
if self.file == '-':
config = yaml.safe_load(sys.stdin)
else:
if os.stat(self.file).st_mode & 0o777 & ~0o600:
raise Exception('refusing to load insecure configuration file, file must have permission 0o600')
with open(self.file) as fp:
config = yaml.safe_load(fp)
if config is None:
config = {}
for k, v in config.items():
self[k] = v
finally:
for keylist, value in self.defaults:
d = self
for i, key in enumerate(keylist):
repl = value if (i == len(keylist) - 1) else {}
d[key] = d.get(key, repl)
d = d[key]
for keylist, value in self.overrides:
d = self
for key in keylist[:-1]:
d[key] = d.get(key, {})
d = d[key]
d[keylist[-1]] = value
def _save(self):
with open(self.file, 'w') as fp:
yaml.safe_dump(dict(self), fp)
def exception(self):
return self._ex
def load(self):
return self._load()
def save(self):
return self._save()
class Config(BaseConfig):
"""
The default configuration.
Keys:
.google.calendar_id: the id of the calendar to sync
.google.credentials: the json of the obtained credentials file
.google.token_file: where to save the login token
.wordpress.url: the base wordpress url
.wordpress.calendar.id: the id of the (wp-booking-system) calendar
.wordpress.calendar.name: the name of the calendar
.wordpress.calendar.translations: a dictionary of language <-> translation pairs (example: {"en": "Reservations"})
.wordpress.credentials.user: the user as which to log into wordpress
.wordpress.credentials.password: the users password
.logging.level: one or more log levels of the form <module.submodule>:<level> seperated by a `,` (comma)
.range: the range in the format <nnn><u>, where n are digits and u is the unit; valid units: 'smhdw' <-> seconds minutes hours days weeks
"""
def __init__(self, file, defaults, overrides, converters, *args, **kwargs):
defaults += [
('google.calendar_id', '#TODO insert google calendar id'),
('google.credentials', {}),
('google.token_file', os.path.join(os.environ['HOME'], '.wp-cal-google-token')),
('wordpress.url', '#TODO insert url to wordpress site'),
('wordpress.calendar.id', '#TODO insert wp-booking-system calendar id'),
('wordpress.calendar.name', '#TODO insert calendar name'),
('wordpress.calendar.translations', {'en': '#TODO insert english translation'}),
('wordpress.credentials.user', '#TODO insert wordpress username'),
('wordpress.credentials.password', '#TODO insert wordpress password'),
('logging.level', ':WARNING,wp_cal:INFO'),
('range', '365d'),
]
defaults = [(x[0].split('.'), x[1]) for x in defaults]
overrides = [(x[0].split('.'), x[1]) for x in overrides]
self.converters = converters
super().__init__(file, defaults, overrides, *args, **kwargs)
def value(self, k):
if isinstance(k, str):
sel = k.split('.')
elif isinstance(k, list):
sel = k
k = '.'.join(k)
else:
raise TypeError('invalid type for parameter k')
v = self
for s in sel:
v = v[s]
if k in self.converters:
return self.converters[k](v)
return v
def range_str_to_timedelta(value): def range_str_to_timedelta(value):
valid_units = 'smhdw' valid_units = 'smhdw'
current_int = 0 current_int = 0
@ -157,12 +44,37 @@ def range_str_to_timedelta(value):
weeks=values['w'] weeks=values['w']
) )
config_schema = Schema({
'sources': [dict],
'sinks': [dict],
Optional('logging', default={'level': ':WARNING,wp_cal:INFO'}): {
Optional('level', default=':WARNING,wp_cal:INFO'): Use(str),
},
Optional('range', default='365d'): Or(
And(
Use(str),
Regex(r'[0-9]+[smhdw]'),
Use(range_str_to_timedelta),
),
lambda x: isinstance(x, datetime.timedelta)
),
})
def load_config(file):
if file == '-':
config = yaml.safe_load(sys.stdin)
else:
if os.stat(file).st_mode & 0o777 & ~0o600:
raise Exception('refusing to load insecure configuration file, file must have permission 0o600')
with open(file) as fp:
config = yaml.safe_load(fp)
return config_schema.validate(config)
def init_logging(): def init_logging():
logging.getLogger().addHandler( logging.getLogger().addHandler(
logging.StreamHandler(), logging.StreamHandler(),
) )
def set_logging_level(level: str): def set_logging_level(level: str):
levels = { levels = {
'NOTSET': logging.NOTSET, 'NOTSET': logging.NOTSET,
@ -191,50 +103,78 @@ def main(config, dryrun, level, range):
init_logging() init_logging()
set_logging_level(':DEBUG') set_logging_level(':DEBUG')
config_overrides = [('logging.level', level), ('range', range)] config = load_config(config)
config_overrides = [x for x in config_overrides if x[1] is not None] if level:
converters = { config['logging']['level'] = level
'range': range_str_to_timedelta, if range:
} config['range'] = range
config = Config(config, [], config_overrides, converters) config = config_schema.validate(config)
if config.exception(): set_logging_level(config['logging']['level'])
logger.info('config not found, trying to generate template')
try: sources = set()
config.save() sinks = set()
except Exception: for source in config['sources']:
logger.exception('failed to generate template') assert len(source.keys()) == 1
for aname, a in adapters.ADAPTERS.items():
if aname in source:
sources |= {a.new(source[aname])}
break
else: else:
logger.info('generated config at "%s"', config.file) logger.error('couldn\'t find valid adapter for source configuration %s', source)
return
set_logging_level(str(config.value('logging.level')))
g = Google(
config.value('google.calendar_id'),
credentials=config.value('google.credentials'),
token_file=config.value('google.token_file'),
)
w = Wordpress(
config.value('wordpress.url'),
calendar_metadata=CalendarMetadata(
id=config.value('wordpress.calendar.id'),
name=config.value('wordpress.calendar.name'),
translations=config.value('wordpress.calendar.translations'),
),
credentials=config.value('wordpress.credentials'),
)
g.login()
events = g.get_events(until=config.value('range'))
logger.info("syncing %d events", len(events))
if not w.login():
logger.info('failed to login to wordpress')
return 1 return 1
for sink in config['sinks']:
assert len(sink.keys()) == 1
for aname, a in adapters.ADAPTERS.items():
if aname in sink:
sinks |= {a.new(sink[aname])}
break
else:
logger.error('couldn\'t find valid adapter for sink configuration %s', sink)
return 1
if not all([isinstance(x, adapters.Source) for x in sources]):
logger.error('one or more source configurations do not implement being a source')
return 1
if not all([isinstance(x, adapters.Sink) for x in sinks]):
logger.error('one or more sink configurations do not implement being a sink')
return 1
# log in
if not all([x.login() for x in sources | sinks]):
logger.error('failed to log into one or more sinks or sources')
return 1
# gather events
events = []
source_results = []
for source in sources:
try:
events += source.get_events(until=config['range'])
source_results += [True]
except Exception:
logger.exception('failed to get events from source %s', source)
source_results += [False]
if not any(source_results):
logger.error('event get failed for all sources')
return 1
logger.info("syncing %d events", len(events))
# post events
if dryrun: if dryrun:
logger.info("dryrun; would post events: %s", events) logger.info("dryrun; would post events: %s", events)
else: else:
if not w.post_events(events, until=config.value('range')): sink_results = []
logger.info('failed to post event data') for sink in sinks:
try:
sink.post_events(events, until=config['range'])
sink_results += [True]
except Exception:
logger.exception('failed to post to sink %s', sink)
sink_results += [False]
if not any(sink_results):
logger.error('event post failed for all sinks')
return 1 return 1
logger.info("done") logger.info("done")
return 0 return 0

View file

@ -1,4 +1,5 @@
pyyaml pyyaml
schema
google-api-python-client google-api-python-client
google-auth-httplib2 google-auth-httplib2
google-auth-oauthlib google-auth-oauthlib