Compare commits

..

1 commit
master ... test

Author SHA1 Message Date
f47cfa6614
Add some simple debug print stmts. 2022-09-28 17:32:36 +02:00
12 changed files with 106 additions and 475 deletions

3
.gitignore vendored
View file

@ -1,4 +1 @@
__pycache__ __pycache__
wp-cal-integration
*.c
*.o

View file

@ -1,12 +0,0 @@
FROM alpine
RUN mkdir -p /opt/wp-cal-integration \
&& apk --no-cache add python3 py3-pip
COPY main.py /opt/wp-cal-integration/
COPY adapters /opt/wp-cal-integration/adapters
COPY requirements.txt /opt/wp-cal-integration/
RUN python3 -m pip install -r /opt/wp-cal-integration/requirements.txt
VOLUME /etc/wp-cal-integration/
ENTRYPOINT [ "/opt/wp-cal-integration/main.py" ]
CMD [ "-c", "/etc/wp-cal-integration/config.yaml" ]

View file

@ -1,21 +0,0 @@
CC ?= cc
CFLAGS ?= $(shell python-config --embed --cflags)
LDFLAGS ?= $(shell python-config --embed --ldflags)
SRC := main.py adapters/*.py
OBJ := $(SRC:%.py=%.o)
wp-cal-integration: $(OBJ)
$(CC) $(LDFLAGS) -o $@ $^
clean:
$(RM) $(OBJ)
main.c: main.py
cython -3 --embed -o $@ $<
%.c: %.py
cython -3 -o $@ $<
%.o: %.c
$(CC) -c $(CFLAGS) -o $@ $<

View file

@ -1,17 +1,5 @@
# Wordpress WP Booking System/Google Calendar integration # Wordpress WP Booking System/Google Calendar integration
## Usage
1. Create a google service account which has permission to read the desired calendars (add it to the calendars).
2. Add a key to the service account (choose the json option) and save it somewhere secure.
3. Run this program once `./main.py --config config.yaml` it will generate a example configuration file.
4. Add the json contents into the example file under `.google.credentials` and fill in the rest of the options.
5. To synchronize once run `./main.py --config config.yaml`.
## CI
There is a concourse pipeline which runs once every hour to synchronize all events. It reads the configuration
file from the concourse vault and passes it to the script.
## Known issues ## Known issues

View file

@ -1,28 +1,11 @@
from .google import Google from .google import Google
from .wordpress import Wordpress from .wordpress import Wordpress
from .abc import Source, Sink, Adapter from .wordpress import CalendarMetadata
_ADAPTERS: set[type[Adapter]] = {
Google,
Wordpress,
}
_SOURCES: set[type[Source]] = {
Google,
}
_SINKS: set[type[Sink]] = {
Wordpress,
}
ADAPTERS: dict[str, type[Adapter]] = {
str(x.__name__).lower(): x for x in _ADAPTERS
}
SOURCES: dict[str, type[Source]] = {
str(x.__name__).lower(): x for x in _SOURCES
}
SINKS: dict[str, type[Sink]] = {
str(x.__name__).lower(): x for x in _SINKS
}
__all__ = [ __all__ = [
'Google', 'Google',
'Wordpress', 'Wordpress',
'CalendarMetadata',
] ]

View file

@ -1,85 +0,0 @@
import abc
import typing
import datetime
import copy
import schema
import dateutil.rrule
class Adapter(abc.ABC):
schema: schema.Schema
def __init__(self, config, *args, **kwargs):
_ = args, kwargs
self.config = config
@classmethod
def new(cls, config: dict, *args, **kwargs):
return cls(cls.schema.validate(config) , *args, **kwargs)
@abc.abstractmethod
def login(self) -> bool:
raise NotImplementedError()
class Source(abc.ABC):
@abc.abstractmethod
def get_events(
self,
start: datetime.datetime | None=None,
until: datetime.timedelta | None=None,
limit=None,
) -> typing.Iterable[dict]:
raise NotImplementedError()
def get_events_resolved(
self,
start: datetime.datetime | None=None,
until: datetime.timedelta | None=None,
limit=None,
) -> typing.Iterable[dict]:
until = until if until else datetime.timedelta(days=365)
now = start if start else datetime.datetime.utcnow().astimezone()
now_365 = now + until
for e in self.get_events(
start=start,
until=until,
limit=limit,
):
if 'recurrence' not in e:
yield e
continue
r = e.pop('recurrence')
r = dateutil.rrule.rrulestr(
'\n'.join(r),
unfold=True,
ignoretz=True,
dtstart=datetime.datetime.fromisoformat(
e['start']['dateTime']
if 'dateTime' in e['start'] else
e['start']['date']
).replace(tzinfo=None)
)
for t_ in r.between(now.replace(tzinfo=None), now_365.replace(tzinfo=None)):
e_ = copy.deepcopy(e)
if 'dateTime' in e['start']:
e_['start']['dateTime'] = datetime.datetime.combine(t_.date(), datetime.datetime.fromisoformat(e['start']['dateTime']).time()).isoformat()
elif 'date' in e['start']:
e_['start']['date'] = datetime.datetime.combine(t_.date(), datetime.time())
if 'dateTime' in e['end']:
e_['end']['dateTime'] = datetime.datetime.combine(t_.date(), datetime.datetime.fromisoformat(e['end']['dateTime']).time()).isoformat()
elif 'date' in e['end']:
e_['end']['date'] = datetime.datetime.combine(t_.date(), datetime.time())
yield e_
yield e
class Sink(abc.ABC):
@abc.abstractmethod
def post_events(
self,
events,
start: datetime.datetime | None=None,
until: datetime.timedelta | None=None,
) -> bool:
raise NotImplementedError()

View file

@ -1,55 +1,52 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import tempfile
import datetime import datetime
import logging import os.path
import os
import json
from google.auth.transport.requests import Request from google.auth.transport.requests import Request
#from google.oauth2.credentials import Credentials from google.oauth2.credentials import Credentials
from google.oauth2.service_account import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build from googleapiclient.discovery import build
from googleapiclient.errors import HttpError from googleapiclient.errors import HttpError
from schema import Use, Optional, Schema
logger = logging.getLogger(f"wp_cal.{__name__}")
from .abc import Source, Adapter
# If modifying these scopes, delete the file token.json. # If modifying these scopes, delete the file token.json.
SCOPES = ['https://www.googleapis.com/auth/calendar.readonly'] SCOPES = ['https://www.googleapis.com/auth/calendar']
class Google(Source, Adapter): class Google():
schema = Schema({ def __init__(self, calendar_id, *, credentials, token_file):
'calendar_id': Use(str), _, credentials_file = tempfile.mkstemp()
'credentials': Use(dict), with open(credentials_file, 'w') as fp:
Optional( json.dump(credentials, fp)
'token_file',
default=os.path.join(
os.environ["HOME"],
'.wp-cal-google-token',
),
): Use(str),
})
def __init__(self, *args, **kwargs): self.calendar_id = calendar_id
super().__init__(*args, **kwargs) self.credentials_file = credentials_file
self.calendar_id = self.config['calendar_id'] self.token_file = token_file
self.credentials_info = self.config['credentials']
self.token_file = self.config['token_file']
# runtime data
self.credentials = None self.credentials = None
def login(self) -> bool: def login(self):
self.credentials = Credentials.from_service_account_info(self.credentials_info, scopes=SCOPES) print('token_file =', self.token_file)
if not self.credentials.valid: print('exists?', os.path.exists(self.token_file))
if os.path.exists(self.token_file):
self.credentials = Credentials.from_authorized_user_file(self.token_file, SCOPES)
print('valid?', self.credentials.valid)
if self.credentials is None or not self.credentials.valid:
if self.credentials is not None \
and self.credentials.expired \
and self.credentials.refresh_token:
self.credentials.refresh(Request()) self.credentials.refresh(Request())
logger.debug('credentials.valid = %s', repr(self.credentials.valid)) else:
return self.credentials.valid flow = InstalledAppFlow.from_client_secrets_file(
self.credentials_file,
SCOPES,
)
self.credentials = flow.run_local_server(port=0)
with open(self.token_file, 'w') as token:
token.write(self.credentials.to_json())
def get_events(self, start: datetime.datetime | None=None, until: datetime.timedelta | None=None, limit=None): def get_events(self, start: datetime.datetime | None=None, until: datetime.timedelta | None=None, limit=None):
until = until if until else datetime.timedelta(days=365) until = until if until else datetime.timedelta(days=365)
@ -65,7 +62,7 @@ class Google(Source, Adapter):
maxResults=limit, maxResults=limit,
).execute() ).execute()
except HttpError: except HttpError:
logger.exception("a http error occured") print('an error occured')
raise raise
events = events.get('items', []) events = events.get('items', [])
return events return events

View file

@ -1,10 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import logging
logger = logging.getLogger(f"wp_cal.{__name__}")
def dict_merge(d0, d1): def dict_merge(d0, d1):
for k, v in d1.items(): for k, v in d1.items():
if (k in d0 and isinstance(d0[k], dict) and isinstance(d1[k], dict)): if (k in d0 and isinstance(d0[k], dict) and isinstance(d1[k], dict)):

View file

@ -5,15 +5,10 @@ import requests
import urllib.parse import urllib.parse
import json import json
import datetime import datetime
import logging
from schema import Use, Schema
from bs4 import BeautifulSoup
from . import utils from . import utils
from .abc import Sink, Adapter
logger = logging.getLogger(f"wp_cal.{__name__}")
class CalendarMetadata(): class CalendarMetadata():
@ -37,29 +32,19 @@ class CalendarMetadata():
**translations, **translations,
} }
class Wordpress(Sink, Adapter): class Wordpress():
schema = Schema({ def __init__(
'url': Use(str), self,
'calendar': { base_url: str,
'id': Use(str), *,
'name': Use(str), calendar_metadata: CalendarMetadata,
'translations': Use(dict), credentials: dict,
}, ):
'credentials': { self.base_url = base_url
'user': Use(str),
'password': Use(str),
},
})
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.base_url = self.config['url']
self.credentials = self.config['credentials']
self.calendar_metadata = CalendarMetadata(**self.config['calendar'])
# runtime data
self.session = requests.Session() self.session = requests.Session()
self.credentials = credentials
self.calendar_meadata = calendar_metadata
def login(self): def login(self):
login_request = self.session.post( login_request = self.session.post(
@ -70,15 +55,9 @@ class Wordpress(Sink, Adapter):
'wp-submit': 'Anmelden', 'wp-submit': 'Anmelden',
'redirect_to': f'{self.base_url}/wp-admin/', 'redirect_to': f'{self.base_url}/wp-admin/',
'testcookie': 1, 'testcookie': 1,
}, }
allow_redirects=False,
) )
logger.debug('login request return headers = %s', login_request.headers) return login_request
login_request_cookies = [x.strip() for x in login_request.headers['Set-Cookie'].split(',')]
if len(login_request_cookies) > 1:
logger.debug('login seems to be ok, cookies = %s', login_request_cookies)
return True
return False
def _datestr_to_date(self, s, tz=None): def _datestr_to_date(self, s, tz=None):
if s.endswith('Z'): if s.endswith('Z'):
@ -138,8 +117,8 @@ class Wordpress(Sink, Adapter):
start: datetime.datetime | None=None, start: datetime.datetime | None=None,
until: datetime.timedelta | None=None until: datetime.timedelta | None=None
): ):
start = start if start else datetime.datetime.utcnow().astimezone()
until = until if until else datetime.timedelta(days=365) until = until if until else datetime.timedelta(days=365)
start = start if start else datetime.datetime.utcnow().astimezone()
final_dict = {} final_dict = {}
for event in events: for event in events:
@ -152,17 +131,9 @@ class Wordpress(Sink, Adapter):
) )
return final_dict return final_dict
def get_nonce(self): def post_events(self, events):
r = self.session.get( metadata = self.calendar_meadata.to_dict()
f'{self.base_url}/wp-admin/admin.php?page=wpbs-calendars&subpage=edit-calendar&calendar_id=1', data = self._generate_data(events)
)
soup = BeautifulSoup(r.text, 'html.parser')
nonce = soup.find_all('input', {'id': 'wpbs_token'})[0]
return nonce['value']
def post_events(self, events, start: datetime.datetime | None=None, until: datetime.timedelta | None=None):
metadata = self.calendar_metadata.to_dict()
data = self._generate_data(events, start=start, until=until)
update_request = self.session.post( update_request = self.session.post(
f'{self.base_url}/wp-admin/admin-ajax.php', f'{self.base_url}/wp-admin/admin-ajax.php',
auth=(self.credentials['user'], self.credentials['password']), auth=(self.credentials['user'], self.credentials['password']),
@ -170,11 +141,7 @@ class Wordpress(Sink, Adapter):
'action': 'wpbs_save_calendar_data', 'action': 'wpbs_save_calendar_data',
'form_data': urllib.parse.urlencode(metadata), 'form_data': urllib.parse.urlencode(metadata),
'calendar_data': json.dumps(data), 'calendar_data': json.dumps(data),
'wpbs_token': self.get_nonce(),
}, },
) )
r = 'wpbs_message=calendar_update_success' in update_request.text return update_request
if not r:
raise Exception(f'failed to post events, got answer {update_request.text}')

View file

@ -1,41 +0,0 @@
---
resources:
- name: periodic
type: time
source:
interval: 24h
- name: script
type: git
source:
uri: https://gitea.redxef.at/redxef/wp-cal-integration
branch: master
jobs:
- name: update-calendar
plan:
- get: periodic
trigger: true
- get: script
- task: run-update
config:
platform: linux
image_resource:
type: registry-image
source:
repository: alpine
tag: latest
inputs:
- name: script
path: .
params:
configuration_json: ((configuration))
run:
path: sh
args:
- -c
- |
#!/usr/bin/env sh
apk --no-cache --update add libcap jq python3 py3-pip
python3 -m pip install --break-system-packages --requirement requirements.txt
echo "$configuration_json" | ./main.py -l :verbose -c -

230
main.py
View file

@ -1,197 +1,67 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import os
import sys import sys
import logging
import datetime
import yaml import yaml
from schema import Schema, And, Or, Use, Optional, Regex try:
from yaml import CLoader as Loader
except ImportError:
from yaml import Loader
import click
import adapters
from adapters import * from adapters import *
logging.VERBOSE = (logging.DEBUG + logging.INFO) // 2 class Config():
logger = logging.getLogger(f'wp_cal') """
The default configuration.
def range_str_to_timedelta(value):
valid_units = 'smhdw'
current_int = 0
values = {}
for c in value:
if c in '0123456789':
current_int *= 10
current_int += int(c)
elif c in valid_units:
if c in values:
logger.warning('unit %s already in values, overwriting', c)
values[c] = current_int
current_int = 0
c = 's'
if current_int != 0:
if c in values:
logger.warning('unit %s already in values, overwriting', c)
values['s'] = current_int
for valid_unit in valid_units:
values.setdefault(valid_unit, 0)
return datetime.timedelta(
seconds=values['s'],
minutes=values['m'],
hours=values['h'],
days=values['d'],
weeks=values['w']
)
def multiple_timedeltas(value): Keys:
mappings = {} .google.calendar_id: the id of the calendar to sync
for v in value.split(','): .google.credentials: the json of the obtained credentials file
kv = v.split(':') .google.token_file: where to save the login token
if len(kv) == 1: .wordpress.url: the base wordpress url
mappings[-1] = range_str_to_timedelta(kv[0]) .wordpress.calendar.id: the id of the (wp-booking-system) calendar
continue .wordpress.calendar.name: the name of the calendar
mappings[int(kv[0])] = range_str_to_timedelta(kv[1]) .wordpress.calendar.translations: a dictionary of language <-> translation pairs (example: {"en": "Reservations"})
return mappings .wordpress.credentials.user: the user as which to log into wordpress
.wordpress.credentials.password: the users password
"""
config_schema = Schema({ def __init__(self, file):
'sources': [dict], self.file = file
'sinks': [dict], self.config: dict | None = None
Optional('logging', default={'level': ':WARNING,wp_cal:INFO'}): {
Optional('level', default=':WARNING,wp_cal:INFO'): Use(str),
},
Optional('range', default='365d'): Or(
dict,
And(
Use(str),
Regex(r'(([0-9]+:)?[0-9]+[smhdw],?)*'),
Use(multiple_timedeltas),
),
),
})
def load_config(file): def load(self):
if file == '-': if self.file == '-':
config = yaml.safe_load(sys.stdin) config = yaml.load(sys.stdin, Loader=Loader)
else: else:
if os.stat(file).st_mode & 0o777 & ~0o600: with open(self.file) as fp:
raise Exception('refusing to load insecure configuration file, file must have permission 0o600') config = yaml.load(fp, Loader=Loader)
with open(file) as fp: self.config = config
config = yaml.safe_load(fp)
return config_schema.validate(config)
def init_logging(): def __getitem__(self, name):
logging.getLogger().addHandler( assert self.config is not None
logging.StreamHandler(), return self.config[name]
)
def set_logging_level(level: str):
levels = {
'NOTSET': logging.NOTSET,
'DEBUG': logging.DEBUG,
'VERBOSE': logging.VERBOSE,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'WARN': logging.WARNING,
'ERROR': logging.ERROR,
'CRITICAL': logging.CRITICAL,
}
log_levels = [x.split(':') for x in level.split(',')]
for module, level in log_levels:
module = module.strip() if module else None
level = level.strip().upper()
if level not in levels:
raise ValueError(f'invalid log level, allowed values: {repr(set(levels.keys()))}')
logging.getLogger(module).setLevel(levels[level])
@click.command()
@click.option('--config', '-c', envvar='WP_CAL_CONFIG', default='-', help='The configuration file')
@click.option('--dryrun', '-d', envvar='WP_CAL_DRYRUN', is_flag=True, help="Don't actually post any data, just show it")
@click.option('--level', '-l', envvar='WP_CAL_LEVEL', default=None, help='The log level for the application')
@click.option('--range', '-r', envvar='WP_CAL_RANGE', default=None, help='The time range from start to start + range to synchronize events')
def main(config, dryrun, level, range):
init_logging()
set_logging_level(':DEBUG')
config = load_config(config)
if level:
config['logging']['level'] = level
if range:
config['range'] = range
config = config_schema.validate(config)
set_logging_level(config['logging']['level'])
sources = set()
sinks = set()
for source in config['sources']:
assert len(source.keys()) == 1
for aname, a in adapters.ADAPTERS.items():
if aname in source:
sources |= {a.new(source[aname])}
break
else:
logger.error('couldn\'t find valid adapter for source configuration %s', source)
sys.exit(1)
for sink in config['sinks']:
assert len(sink.keys()) == 1
for aname, a in adapters.ADAPTERS.items():
if aname in sink:
sinks |= {a.new(sink[aname])}
break
else:
logger.error('couldn\'t find valid adapter for sink configuration %s', sink)
sys.exit(1)
if not all([isinstance(x, adapters.Source) for x in sources]):
logger.error('one or more source configurations do not implement being a source')
sys.exit(1)
if not all([isinstance(x, adapters.Sink) for x in sinks]):
logger.error('one or more sink configurations do not implement being a sink')
sys.exit(1)
# log in
if not all([x.login() for x in sources | sinks]):
logger.error('failed to log into one or more sinks or sources')
sys.exit(1)
# gather events
events = []
source_results = []
for i, source in enumerate(sources):
try:
events += source.get_events_resolved(until=config['range'].get(i, config['range'][-1]))
source_results += [True]
except Exception:
logger.exception('failed to get events from source %s', source)
source_results += [False]
if not any(source_results):
logger.error('event get failed for all sources')
sys.exit(1)
# filter cancelled events
logger.info('found %d events', len(events))
logger.info('not syncing cancelled events')
events = [e for e in events if e['status'] != 'cancelled']
logger.info('syncing %d events', len(events))
logger.log(logging.VERBOSE, 'events are: %s', [f"{e.get('summary', '<NO SUMMARY PROVIDED>')} ({e.get('start')})" for e in events])
# post events
if not dryrun:
sink_results = []
for sink in sinks:
try:
sink.post_events(events, until=max(config['range'].values()))
sink_results += [True]
except Exception:
logger.exception('failed to post to sink %s', sink)
sink_results += [False]
if not any(sink_results):
logger.error('event post failed for all sinks')
sys.exit(1)
logger.info("done")
sys.exit(0)
if __name__ == '__main__': if __name__ == '__main__':
main() config = Config('-')
config.load()
g = Google(
config['google']['calendar_id'],
credentials=config['google']['credentials'],
token_file=config['google'].get('token_file', '~/.wp-cal-integration-google-token.json')
)
w = Wordpress(
config['wordpress']['url'],
calendar_metadata=CalendarMetadata(
id=config['wordpress']['calendar']['id'],
name=config['wordpress']['calendar']['name'],
translations=config['wordpress']['calendar']['translations'],
),
credentials=config['wordpress']['credentials'],
)
g.login()
events = g.get_events()
w.login()
w.post_events(events)

View file

@ -1,8 +0,0 @@
pyyaml
schema
google-api-python-client
google-auth-httplib2
google-auth-oauthlib
click
bs4
python-dateutil