Switch config validation. Allow multiple sources/sinks.
This commit is contained in:
parent
d08fea10b8
commit
a1ae079e8a
GPG key ID:
7DAC3AA211CBD921
6 changed files with 214 additions and 179 deletions
|
|
@ -1,11 +1,28 @@
|
|||
|
||||
|
||||
from .google import Google
|
||||
from .wordpress import Wordpress
|
||||
from .wordpress import CalendarMetadata
|
||||
from .abc import Source, Sink, Adapter
|
||||
|
||||
_ADAPTERS: set[type[Adapter]] = {
|
||||
Google,
|
||||
Wordpress,
|
||||
}
|
||||
_SOURCES: set[type[Source]] = {
|
||||
Google,
|
||||
}
|
||||
_SINKS: set[type[Sink]] = {
|
||||
Wordpress,
|
||||
}
|
||||
ADAPTERS: dict[str, type[Adapter]] = {
|
||||
str(x.__name__).lower(): x for x in _ADAPTERS
|
||||
}
|
||||
SOURCES: dict[str, type[Source]] = {
|
||||
str(x.__name__).lower(): x for x in _SOURCES
|
||||
}
|
||||
SINKS: dict[str, type[Sink]] = {
|
||||
str(x.__name__).lower(): x for x in _SINKS
|
||||
}
|
||||
|
||||
__all__ = [
|
||||
'Google',
|
||||
'Wordpress',
|
||||
'CalendarMetadata',
|
||||
]
|
||||
|
|
|
|||
42
adapters/abc.py
Normal file
42
adapters/abc.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
import abc
|
||||
import typing
|
||||
import datetime
|
||||
|
||||
import schema
|
||||
|
||||
class Adapter(abc.ABC):
|
||||
schema: schema.Schema
|
||||
|
||||
def __init__(self, config, *args, **kwargs):
|
||||
_ = args, kwargs
|
||||
self.config = config
|
||||
|
||||
@classmethod
|
||||
def new(cls, config: dict, *args, **kwargs):
|
||||
return cls(cls.schema.validate(config) , *args, **kwargs)
|
||||
|
||||
@abc.abstractmethod
|
||||
def login(self) -> bool:
|
||||
raise NotImplementedError()
|
||||
|
||||
class Source(abc.ABC):
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_events(
|
||||
self,
|
||||
start: datetime.datetime | None=None,
|
||||
until: datetime.timedelta | None=None,
|
||||
limit=None,
|
||||
) -> typing.Iterable[dict]:
|
||||
raise NotImplementedError()
|
||||
|
||||
class Sink(abc.ABC):
|
||||
|
||||
@abc.abstractmethod
|
||||
def post_events(
|
||||
self,
|
||||
events,
|
||||
start: datetime.datetime | None=None,
|
||||
until: datetime.timedelta | None=None,
|
||||
) -> bool:
|
||||
raise NotImplementedError()
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
|
||||
from google.auth.transport.requests import Request
|
||||
#from google.oauth2.credentials import Credentials
|
||||
|
|
@ -10,27 +11,45 @@ from google.oauth2.service_account import Credentials
|
|||
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||
from googleapiclient.discovery import build
|
||||
from googleapiclient.errors import HttpError
|
||||
from schema import Use, Optional, Schema
|
||||
|
||||
logger = logging.getLogger(f"wp_cal.{__name__}")
|
||||
print(logger.name)
|
||||
|
||||
from .abc import Source, Adapter
|
||||
|
||||
# If modifying these scopes, delete the file token.json.
|
||||
SCOPES = ['https://www.googleapis.com/auth/calendar.readonly']
|
||||
|
||||
|
||||
class Google():
|
||||
class Google(Source, Adapter):
|
||||
|
||||
def __init__(self, calendar_id, *, credentials, token_file):
|
||||
self.calendar_id = calendar_id
|
||||
self.credentials_info = credentials
|
||||
schema = Schema({
|
||||
'calendar_id': Use(str),
|
||||
'credentials': Use(dict),
|
||||
Optional(
|
||||
'token_file',
|
||||
default=os.path.join(
|
||||
os.environ["HOME"],
|
||||
'.wp-cal-google-token',
|
||||
),
|
||||
): Use(str),
|
||||
})
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.calendar_id = self.config['calendar_id']
|
||||
self.credentials_info = self.config['credentials']
|
||||
self.token_file = self.config['token_file']
|
||||
|
||||
# runtime data
|
||||
self.credentials = None
|
||||
|
||||
def login(self):
|
||||
def login(self) -> bool:
|
||||
self.credentials = Credentials.from_service_account_info(self.credentials_info, scopes=SCOPES)
|
||||
if not self.credentials.valid:
|
||||
self.credentials.refresh(Request())
|
||||
logger.debug('credentials.valid = %s', repr(self.credentials.valid))
|
||||
|
||||
return self.credentials.valid
|
||||
|
||||
def get_events(self, start: datetime.datetime | None=None, until: datetime.timedelta | None=None, limit=None):
|
||||
until = until if until else datetime.timedelta(days=365)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,10 @@ import json
|
|||
import datetime
|
||||
import logging
|
||||
|
||||
from schema import Use, Schema
|
||||
|
||||
from . import utils
|
||||
from .abc import Sink, Adapter
|
||||
|
||||
logger = logging.getLogger(f"wp_cal.{__name__}")
|
||||
|
||||
|
|
@ -33,19 +36,29 @@ class CalendarMetadata():
|
|||
**translations,
|
||||
}
|
||||
|
||||
class Wordpress():
|
||||
class Wordpress(Sink, Adapter):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
*,
|
||||
calendar_metadata: CalendarMetadata,
|
||||
credentials: dict,
|
||||
):
|
||||
self.base_url = base_url
|
||||
schema = Schema({
|
||||
'url': Use(str),
|
||||
'calendar': {
|
||||
'id': Use(str),
|
||||
'name': Use(str),
|
||||
'translations': Use(dict),
|
||||
},
|
||||
'credentials': {
|
||||
'user': Use(str),
|
||||
'password': Use(str),
|
||||
},
|
||||
})
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.base_url = self.config['url']
|
||||
self.credentials = self.config['credentials']
|
||||
self.calendar_metadata = CalendarMetadata(**self.config['calendar'])
|
||||
|
||||
# runtime data
|
||||
self.session = requests.Session()
|
||||
self.credentials = credentials
|
||||
self.calendar_meadata = calendar_metadata
|
||||
|
||||
def login(self):
|
||||
login_request = self.session.post(
|
||||
|
|
@ -139,7 +152,7 @@ class Wordpress():
|
|||
return final_dict
|
||||
|
||||
def post_events(self, events, start: datetime.datetime | None=None, until: datetime.timedelta | None=None):
|
||||
metadata = self.calendar_meadata.to_dict()
|
||||
metadata = self.calendar_metadata.to_dict()
|
||||
data = self._generate_data(events, start=start, until=until)
|
||||
update_request = self.session.post(
|
||||
f'{self.base_url}/wp-admin/admin-ajax.php',
|
||||
|
|
@ -150,5 +163,8 @@ class Wordpress():
|
|||
'calendar_data': json.dumps(data),
|
||||
},
|
||||
)
|
||||
return 'wpbs_message=calendar_update_success' in update_request.text
|
||||
r = 'wpbs_message=calendar_update_success' in update_request.text
|
||||
if not r:
|
||||
raise Exception('failed to post events')
|
||||
|
||||
|
||||
|
|
|
|||
252
main.py
252
main.py
|
|
@ -1,134 +1,21 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import functools
|
||||
import os
|
||||
import sys
|
||||
import yaml
|
||||
import logging
|
||||
import datetime
|
||||
try:
|
||||
from yaml import CLoader as Loader, CDumper as Dumper
|
||||
except ImportError:
|
||||
from yaml import Loader, Dumper
|
||||
import yaml
|
||||
from schema import Schema, And, Or, Use, Optional, Regex
|
||||
|
||||
import click
|
||||
|
||||
import adapters
|
||||
from adapters import *
|
||||
|
||||
logger = logging.getLogger(f'wp_cal')
|
||||
|
||||
class BaseConfig(dict):
|
||||
|
||||
def __init__(self, file, defaults, overrides, *args, **kwargs):
|
||||
self._ex = None
|
||||
self.file = file
|
||||
self.defaults = defaults
|
||||
self.overrides = overrides
|
||||
super().__init__(*args, **kwargs)
|
||||
try:
|
||||
self._load()
|
||||
except FileNotFoundError as ex:
|
||||
self._ex = ex
|
||||
|
||||
def __repr__(self):
|
||||
return super().__repr__()
|
||||
|
||||
def __str__(self):
|
||||
return super().__str__()
|
||||
|
||||
def _load(self):
|
||||
try:
|
||||
if self.file == '-':
|
||||
config = yaml.safe_load(sys.stdin)
|
||||
else:
|
||||
if os.stat(self.file).st_mode & 0o777 & ~0o600:
|
||||
raise Exception('refusing to load insecure configuration file, file must have permission 0o600')
|
||||
with open(self.file) as fp:
|
||||
config = yaml.safe_load(fp)
|
||||
if config is None:
|
||||
config = {}
|
||||
for k, v in config.items():
|
||||
self[k] = v
|
||||
finally:
|
||||
for keylist, value in self.defaults:
|
||||
d = self
|
||||
for i, key in enumerate(keylist):
|
||||
repl = value if (i == len(keylist) - 1) else {}
|
||||
d[key] = d.get(key, repl)
|
||||
d = d[key]
|
||||
for keylist, value in self.overrides:
|
||||
d = self
|
||||
for key in keylist[:-1]:
|
||||
d[key] = d.get(key, {})
|
||||
d = d[key]
|
||||
d[keylist[-1]] = value
|
||||
|
||||
def _save(self):
|
||||
with open(self.file, 'w') as fp:
|
||||
yaml.safe_dump(dict(self), fp)
|
||||
|
||||
def exception(self):
|
||||
return self._ex
|
||||
|
||||
def load(self):
|
||||
return self._load()
|
||||
|
||||
def save(self):
|
||||
return self._save()
|
||||
|
||||
class Config(BaseConfig):
|
||||
"""
|
||||
The default configuration.
|
||||
|
||||
|
||||
Keys:
|
||||
.google.calendar_id: the id of the calendar to sync
|
||||
.google.credentials: the json of the obtained credentials file
|
||||
.google.token_file: where to save the login token
|
||||
.wordpress.url: the base wordpress url
|
||||
.wordpress.calendar.id: the id of the (wp-booking-system) calendar
|
||||
.wordpress.calendar.name: the name of the calendar
|
||||
.wordpress.calendar.translations: a dictionary of language <-> translation pairs (example: {"en": "Reservations"})
|
||||
.wordpress.credentials.user: the user as which to log into wordpress
|
||||
.wordpress.credentials.password: the users password
|
||||
.logging.level: one or more log levels of the form <module.submodule>:<level> seperated by a `,` (comma)
|
||||
.range: the range in the format <nnn><u>, where n are digits and u is the unit; valid units: 'smhdw' <-> seconds minutes hours days weeks
|
||||
"""
|
||||
|
||||
def __init__(self, file, defaults, overrides, converters, *args, **kwargs):
|
||||
defaults += [
|
||||
('google.calendar_id', '#TODO insert google calendar id'),
|
||||
('google.credentials', {}),
|
||||
('google.token_file', os.path.join(os.environ['HOME'], '.wp-cal-google-token')),
|
||||
('wordpress.url', '#TODO insert url to wordpress site'),
|
||||
('wordpress.calendar.id', '#TODO insert wp-booking-system calendar id'),
|
||||
('wordpress.calendar.name', '#TODO insert calendar name'),
|
||||
('wordpress.calendar.translations', {'en': '#TODO insert english translation'}),
|
||||
('wordpress.credentials.user', '#TODO insert wordpress username'),
|
||||
('wordpress.credentials.password', '#TODO insert wordpress password'),
|
||||
('logging.level', ':WARNING,wp_cal:INFO'),
|
||||
('range', '365d'),
|
||||
]
|
||||
defaults = [(x[0].split('.'), x[1]) for x in defaults]
|
||||
overrides = [(x[0].split('.'), x[1]) for x in overrides]
|
||||
self.converters = converters
|
||||
super().__init__(file, defaults, overrides, *args, **kwargs)
|
||||
|
||||
def value(self, k):
|
||||
if isinstance(k, str):
|
||||
sel = k.split('.')
|
||||
elif isinstance(k, list):
|
||||
sel = k
|
||||
k = '.'.join(k)
|
||||
else:
|
||||
raise TypeError('invalid type for parameter k')
|
||||
v = self
|
||||
for s in sel:
|
||||
v = v[s]
|
||||
if k in self.converters:
|
||||
return self.converters[k](v)
|
||||
return v
|
||||
|
||||
def range_str_to_timedelta(value):
|
||||
valid_units = 'smhdw'
|
||||
current_int = 0
|
||||
|
|
@ -157,12 +44,37 @@ def range_str_to_timedelta(value):
|
|||
weeks=values['w']
|
||||
)
|
||||
|
||||
config_schema = Schema({
|
||||
'sources': [dict],
|
||||
'sinks': [dict],
|
||||
Optional('logging', default={'level': ':WARNING,wp_cal:INFO'}): {
|
||||
Optional('level', default=':WARNING,wp_cal:INFO'): Use(str),
|
||||
},
|
||||
Optional('range', default='365d'): Or(
|
||||
And(
|
||||
Use(str),
|
||||
Regex(r'[0-9]+[smhdw]'),
|
||||
Use(range_str_to_timedelta),
|
||||
),
|
||||
lambda x: isinstance(x, datetime.timedelta)
|
||||
),
|
||||
})
|
||||
|
||||
def load_config(file):
|
||||
if file == '-':
|
||||
config = yaml.safe_load(sys.stdin)
|
||||
else:
|
||||
if os.stat(file).st_mode & 0o777 & ~0o600:
|
||||
raise Exception('refusing to load insecure configuration file, file must have permission 0o600')
|
||||
with open(file) as fp:
|
||||
config = yaml.safe_load(fp)
|
||||
return config_schema.validate(config)
|
||||
|
||||
def init_logging():
|
||||
logging.getLogger().addHandler(
|
||||
logging.StreamHandler(),
|
||||
)
|
||||
|
||||
|
||||
def set_logging_level(level: str):
|
||||
levels = {
|
||||
'NOTSET': logging.NOTSET,
|
||||
|
|
@ -191,50 +103,78 @@ def main(config, dryrun, level, range):
|
|||
init_logging()
|
||||
set_logging_level(':DEBUG')
|
||||
|
||||
config_overrides = [('logging.level', level), ('range', range)]
|
||||
config_overrides = [x for x in config_overrides if x[1] is not None]
|
||||
converters = {
|
||||
'range': range_str_to_timedelta,
|
||||
}
|
||||
config = Config(config, [], config_overrides, converters)
|
||||
if config.exception():
|
||||
logger.info('config not found, trying to generate template')
|
||||
try:
|
||||
config.save()
|
||||
except Exception:
|
||||
logger.exception('failed to generate template')
|
||||
config = load_config(config)
|
||||
if level:
|
||||
config['logging']['level'] = level
|
||||
if range:
|
||||
config['range'] = range
|
||||
config = config_schema.validate(config)
|
||||
set_logging_level(config['logging']['level'])
|
||||
|
||||
sources = set()
|
||||
sinks = set()
|
||||
for source in config['sources']:
|
||||
assert len(source.keys()) == 1
|
||||
for aname, a in adapters.ADAPTERS.items():
|
||||
if aname in source:
|
||||
sources |= {a.new(source[aname])}
|
||||
break
|
||||
else:
|
||||
logger.info('generated config at "%s"', config.file)
|
||||
return
|
||||
|
||||
set_logging_level(str(config.value('logging.level')))
|
||||
|
||||
g = Google(
|
||||
config.value('google.calendar_id'),
|
||||
credentials=config.value('google.credentials'),
|
||||
token_file=config.value('google.token_file'),
|
||||
)
|
||||
w = Wordpress(
|
||||
config.value('wordpress.url'),
|
||||
calendar_metadata=CalendarMetadata(
|
||||
id=config.value('wordpress.calendar.id'),
|
||||
name=config.value('wordpress.calendar.name'),
|
||||
translations=config.value('wordpress.calendar.translations'),
|
||||
),
|
||||
credentials=config.value('wordpress.credentials'),
|
||||
)
|
||||
g.login()
|
||||
events = g.get_events(until=config.value('range'))
|
||||
logger.info("syncing %d events", len(events))
|
||||
if not w.login():
|
||||
logger.info('failed to login to wordpress')
|
||||
logger.error('couldn\'t find valid adapter for source configuration %s', source)
|
||||
return 1
|
||||
for sink in config['sinks']:
|
||||
assert len(sink.keys()) == 1
|
||||
for aname, a in adapters.ADAPTERS.items():
|
||||
if aname in sink:
|
||||
sinks |= {a.new(sink[aname])}
|
||||
break
|
||||
else:
|
||||
logger.error('couldn\'t find valid adapter for sink configuration %s', sink)
|
||||
return 1
|
||||
|
||||
if not all([isinstance(x, adapters.Source) for x in sources]):
|
||||
logger.error('one or more source configurations do not implement being a source')
|
||||
return 1
|
||||
if not all([isinstance(x, adapters.Sink) for x in sinks]):
|
||||
logger.error('one or more sink configurations do not implement being a sink')
|
||||
return 1
|
||||
|
||||
# log in
|
||||
if not all([x.login() for x in sources | sinks]):
|
||||
logger.error('failed to log into one or more sinks or sources')
|
||||
return 1
|
||||
|
||||
# gather events
|
||||
events = []
|
||||
source_results = []
|
||||
for source in sources:
|
||||
try:
|
||||
events += source.get_events(until=config['range'])
|
||||
source_results += [True]
|
||||
except Exception:
|
||||
logger.exception('failed to get events from source %s', source)
|
||||
source_results += [False]
|
||||
if not any(source_results):
|
||||
logger.error('event get failed for all sources')
|
||||
return 1
|
||||
logger.info("syncing %d events", len(events))
|
||||
|
||||
# post events
|
||||
if dryrun:
|
||||
logger.info("dryrun; would post events: %s", events)
|
||||
else:
|
||||
if not w.post_events(events, until=config.value('range')):
|
||||
logger.info('failed to post event data')
|
||||
sink_results = []
|
||||
for sink in sinks:
|
||||
try:
|
||||
sink.post_events(events, until=config['range'])
|
||||
sink_results += [True]
|
||||
except Exception:
|
||||
logger.exception('failed to post to sink %s', sink)
|
||||
sink_results += [False]
|
||||
if not any(sink_results):
|
||||
logger.error('event post failed for all sinks')
|
||||
return 1
|
||||
|
||||
logger.info("done")
|
||||
return 0
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
pyyaml
|
||||
schema
|
||||
google-api-python-client
|
||||
google-auth-httplib2
|
||||
google-auth-oauthlib
|
||||
|
|
|
|||
Loading…
Reference in a new issue