Compare commits

...

29 commits
test ... master

Author SHA1 Message Date
41d6c4be84
Update only every day. 2024-03-16 10:52:53 +01:00
aee3d1fc91
Add break-system-packages until we improve packaging. 2023-12-11 13:50:25 +01:00
7146fc2932
Fix range on upload. 2023-11-14 18:47:22 +01:00
c892ac1aab
Allow different time ranges for the different sources. 2023-11-14 18:42:55 +01:00
264301ecc0
Fix date generation if no start is specified in the repeat rules. 2023-11-14 18:11:58 +01:00
74cd1f91b4
Use verbose logging in pipeline. 2023-11-14 16:00:43 +01:00
89b769088e
Add verbose log level. 2023-11-14 15:59:57 +01:00
437ad585ac
Add nonce support. 2023-11-14 15:40:53 +01:00
1e2137cd90
Include api response text in exception body. 2023-06-15 03:07:18 +02:00
04a1e36a00
Fix exit codes. 2023-02-18 21:04:20 +01:00
5e0ff701da
Fix sync if it contains cancelled events. 2023-02-18 21:01:05 +01:00
a1ae079e8a
Switch config validation. Allow multiple sources/sinks. 2023-02-18 20:48:22 +01:00
d08fea10b8
Explicit latest tag in pipeline.
Alpine edge failed with python3 no module pip, after py3-pip
being installed.
2022-11-23 14:55:57 +01:00
25b5ff81fc
Add dockerfile. 2022-11-23 02:38:10 +01:00
d629e8fb71
Add initial cython makefile. 2022-11-23 02:00:36 +01:00
605abaf177
Fix template generation. 2022-11-23 02:00:16 +01:00
fd8ed776d4
Update readme. 2022-10-13 11:49:44 +02:00
3b8ff2d331
Change pipeline for new method. 2022-10-13 11:47:43 +02:00
df0beec957
Default logging level on startup. 2022-10-13 11:45:03 +02:00
39923a0c5f
Change config loading method. 2022-10-13 11:27:37 +02:00
69b11d513a
Add range parameter. 2022-10-13 11:06:44 +02:00
8460f70216
Add file permission check on config. 2022-10-13 01:16:37 +02:00
027c4737cb
Improve readme. 2022-10-13 01:08:40 +02:00
a68512cbb4
Add requirements.txt. 2022-10-13 00:57:40 +02:00
d3895cd43d
Add dryrun flag. 2022-10-13 00:53:28 +02:00
4059b9b6dd
Remove unused imports. 2022-10-13 00:47:59 +02:00
21be48f8e3
Change google adapter to use service account. 2022-10-13 00:47:08 +02:00
e683a90b34
Improve logging and config loading. 2022-10-13 00:16:05 +02:00
f6d97fe790
Add ci pipeline, doesn't really work because of oauth. 2022-09-28 17:55:27 +02:00
12 changed files with 475 additions and 103 deletions

3
.gitignore vendored
View file

@ -1 +1,4 @@
__pycache__ __pycache__
wp-cal-integration
*.c
*.o

12
Dockerfile Normal file
View file

@ -0,0 +1,12 @@
FROM alpine
RUN mkdir -p /opt/wp-cal-integration \
&& apk --no-cache add python3 py3-pip
COPY main.py /opt/wp-cal-integration/
COPY adapters /opt/wp-cal-integration/adapters
COPY requirements.txt /opt/wp-cal-integration/
RUN python3 -m pip install -r /opt/wp-cal-integration/requirements.txt
VOLUME /etc/wp-cal-integration/
ENTRYPOINT [ "/opt/wp-cal-integration/main.py" ]
CMD [ "-c", "/etc/wp-cal-integration/config.yaml" ]

21
Makefile Normal file
View file

@ -0,0 +1,21 @@
CC ?= cc
CFLAGS ?= $(shell python-config --embed --cflags)
LDFLAGS ?= $(shell python-config --embed --ldflags)
SRC := main.py adapters/*.py
OBJ := $(SRC:%.py=%.o)
wp-cal-integration: $(OBJ)
$(CC) $(LDFLAGS) -o $@ $^
clean:
$(RM) $(OBJ)
main.c: main.py
cython -3 --embed -o $@ $<
%.c: %.py
cython -3 -o $@ $<
%.o: %.c
$(CC) -c $(CFLAGS) -o $@ $<

View file

@ -1,5 +1,17 @@
# Wordpress WP Booking System/Google Calendar integration # Wordpress WP Booking System/Google Calendar integration
## Usage
1. Create a google service account which has permission to read the desired calendars (add it to the calendars).
2. Add a key to the service account (choose the json option) and save it somewhere secure.
3. Run this program once `./main.py --config config.yaml` it will generate a example configuration file.
4. Add the json contents into the example file under `.google.credentials` and fill in the rest of the options.
5. To synchronize once run `./main.py --config config.yaml`.
## CI
There is a concourse pipeline which runs once every hour to synchronize all events. It reads the configuration
file from the concourse vault and passes it to the script.
## Known issues ## Known issues

View file

@ -1,11 +1,28 @@
from .google import Google from .google import Google
from .wordpress import Wordpress from .wordpress import Wordpress
from .wordpress import CalendarMetadata from .abc import Source, Sink, Adapter
_ADAPTERS: set[type[Adapter]] = {
Google,
Wordpress,
}
_SOURCES: set[type[Source]] = {
Google,
}
_SINKS: set[type[Sink]] = {
Wordpress,
}
ADAPTERS: dict[str, type[Adapter]] = {
str(x.__name__).lower(): x for x in _ADAPTERS
}
SOURCES: dict[str, type[Source]] = {
str(x.__name__).lower(): x for x in _SOURCES
}
SINKS: dict[str, type[Sink]] = {
str(x.__name__).lower(): x for x in _SINKS
}
__all__ = [ __all__ = [
'Google', 'Google',
'Wordpress', 'Wordpress',
'CalendarMetadata',
] ]

85
adapters/abc.py Normal file
View file

@ -0,0 +1,85 @@
import abc
import typing
import datetime
import copy
import schema
import dateutil.rrule
class Adapter(abc.ABC):
schema: schema.Schema
def __init__(self, config, *args, **kwargs):
_ = args, kwargs
self.config = config
@classmethod
def new(cls, config: dict, *args, **kwargs):
return cls(cls.schema.validate(config) , *args, **kwargs)
@abc.abstractmethod
def login(self) -> bool:
raise NotImplementedError()
class Source(abc.ABC):
@abc.abstractmethod
def get_events(
self,
start: datetime.datetime | None=None,
until: datetime.timedelta | None=None,
limit=None,
) -> typing.Iterable[dict]:
raise NotImplementedError()
def get_events_resolved(
self,
start: datetime.datetime | None=None,
until: datetime.timedelta | None=None,
limit=None,
) -> typing.Iterable[dict]:
until = until if until else datetime.timedelta(days=365)
now = start if start else datetime.datetime.utcnow().astimezone()
now_365 = now + until
for e in self.get_events(
start=start,
until=until,
limit=limit,
):
if 'recurrence' not in e:
yield e
continue
r = e.pop('recurrence')
r = dateutil.rrule.rrulestr(
'\n'.join(r),
unfold=True,
ignoretz=True,
dtstart=datetime.datetime.fromisoformat(
e['start']['dateTime']
if 'dateTime' in e['start'] else
e['start']['date']
).replace(tzinfo=None)
)
for t_ in r.between(now.replace(tzinfo=None), now_365.replace(tzinfo=None)):
e_ = copy.deepcopy(e)
if 'dateTime' in e['start']:
e_['start']['dateTime'] = datetime.datetime.combine(t_.date(), datetime.datetime.fromisoformat(e['start']['dateTime']).time()).isoformat()
elif 'date' in e['start']:
e_['start']['date'] = datetime.datetime.combine(t_.date(), datetime.time())
if 'dateTime' in e['end']:
e_['end']['dateTime'] = datetime.datetime.combine(t_.date(), datetime.datetime.fromisoformat(e['end']['dateTime']).time()).isoformat()
elif 'date' in e['end']:
e_['end']['date'] = datetime.datetime.combine(t_.date(), datetime.time())
yield e_
yield e
class Sink(abc.ABC):
@abc.abstractmethod
def post_events(
self,
events,
start: datetime.datetime | None=None,
until: datetime.timedelta | None=None,
) -> bool:
raise NotImplementedError()

View file

@ -1,49 +1,55 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import tempfile
import datetime import datetime
import os.path import logging
import os
import json
from google.auth.transport.requests import Request from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials #from google.oauth2.credentials import Credentials
from google.oauth2.service_account import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build from googleapiclient.discovery import build
from googleapiclient.errors import HttpError from googleapiclient.errors import HttpError
from schema import Use, Optional, Schema
logger = logging.getLogger(f"wp_cal.{__name__}")
from .abc import Source, Adapter
# If modifying these scopes, delete the file token.json. # If modifying these scopes, delete the file token.json.
SCOPES = ['https://www.googleapis.com/auth/calendar'] SCOPES = ['https://www.googleapis.com/auth/calendar.readonly']
class Google(): class Google(Source, Adapter):
def __init__(self, calendar_id, *, credentials, token_file): schema = Schema({
_, credentials_file = tempfile.mkstemp() 'calendar_id': Use(str),
with open(credentials_file, 'w') as fp: 'credentials': Use(dict),
json.dump(credentials, fp) Optional(
'token_file',
default=os.path.join(
os.environ["HOME"],
'.wp-cal-google-token',
),
): Use(str),
})
self.calendar_id = calendar_id def __init__(self, *args, **kwargs):
self.credentials_file = credentials_file super().__init__(*args, **kwargs)
self.token_file = token_file self.calendar_id = self.config['calendar_id']
self.credentials_info = self.config['credentials']
self.token_file = self.config['token_file']
# runtime data
self.credentials = None self.credentials = None
def login(self): def login(self) -> bool:
if os.path.exists(self.token_file): self.credentials = Credentials.from_service_account_info(self.credentials_info, scopes=SCOPES)
self.credentials = Credentials.from_authorized_user_file(self.token_file, SCOPES) if not self.credentials.valid:
if self.credentials is None or not self.credentials.valid:
if self.credentials is not None \
and self.credentials.expired \
and self.credentials.refresh_token:
self.credentials.refresh(Request()) self.credentials.refresh(Request())
else: logger.debug('credentials.valid = %s', repr(self.credentials.valid))
flow = InstalledAppFlow.from_client_secrets_file( return self.credentials.valid
self.credentials_file,
SCOPES,
)
self.credentials = flow.run_local_server(port=0)
with open(self.token_file, 'w') as token:
token.write(self.credentials.to_json())
def get_events(self, start: datetime.datetime | None=None, until: datetime.timedelta | None=None, limit=None): def get_events(self, start: datetime.datetime | None=None, until: datetime.timedelta | None=None, limit=None):
until = until if until else datetime.timedelta(days=365) until = until if until else datetime.timedelta(days=365)
@ -59,7 +65,7 @@ class Google():
maxResults=limit, maxResults=limit,
).execute() ).execute()
except HttpError: except HttpError:
print('an error occured') logger.exception("a http error occured")
raise raise
events = events.get('items', []) events = events.get('items', [])
return events return events

View file

@ -1,6 +1,10 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import logging
logger = logging.getLogger(f"wp_cal.{__name__}")
def dict_merge(d0, d1): def dict_merge(d0, d1):
for k, v in d1.items(): for k, v in d1.items():
if (k in d0 and isinstance(d0[k], dict) and isinstance(d1[k], dict)): if (k in d0 and isinstance(d0[k], dict) and isinstance(d1[k], dict)):

View file

@ -5,10 +5,15 @@ import requests
import urllib.parse import urllib.parse
import json import json
import datetime import datetime
import logging
from schema import Use, Schema
from bs4 import BeautifulSoup
from . import utils from . import utils
from .abc import Sink, Adapter
logger = logging.getLogger(f"wp_cal.{__name__}")
class CalendarMetadata(): class CalendarMetadata():
@ -32,19 +37,29 @@ class CalendarMetadata():
**translations, **translations,
} }
class Wordpress(): class Wordpress(Sink, Adapter):
def __init__( schema = Schema({
self, 'url': Use(str),
base_url: str, 'calendar': {
*, 'id': Use(str),
calendar_metadata: CalendarMetadata, 'name': Use(str),
credentials: dict, 'translations': Use(dict),
): },
self.base_url = base_url 'credentials': {
'user': Use(str),
'password': Use(str),
},
})
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.base_url = self.config['url']
self.credentials = self.config['credentials']
self.calendar_metadata = CalendarMetadata(**self.config['calendar'])
# runtime data
self.session = requests.Session() self.session = requests.Session()
self.credentials = credentials
self.calendar_meadata = calendar_metadata
def login(self): def login(self):
login_request = self.session.post( login_request = self.session.post(
@ -55,9 +70,15 @@ class Wordpress():
'wp-submit': 'Anmelden', 'wp-submit': 'Anmelden',
'redirect_to': f'{self.base_url}/wp-admin/', 'redirect_to': f'{self.base_url}/wp-admin/',
'testcookie': 1, 'testcookie': 1,
} },
allow_redirects=False,
) )
return login_request logger.debug('login request return headers = %s', login_request.headers)
login_request_cookies = [x.strip() for x in login_request.headers['Set-Cookie'].split(',')]
if len(login_request_cookies) > 1:
logger.debug('login seems to be ok, cookies = %s', login_request_cookies)
return True
return False
def _datestr_to_date(self, s, tz=None): def _datestr_to_date(self, s, tz=None):
if s.endswith('Z'): if s.endswith('Z'):
@ -117,8 +138,8 @@ class Wordpress():
start: datetime.datetime | None=None, start: datetime.datetime | None=None,
until: datetime.timedelta | None=None until: datetime.timedelta | None=None
): ):
until = until if until else datetime.timedelta(days=365)
start = start if start else datetime.datetime.utcnow().astimezone() start = start if start else datetime.datetime.utcnow().astimezone()
until = until if until else datetime.timedelta(days=365)
final_dict = {} final_dict = {}
for event in events: for event in events:
@ -131,9 +152,17 @@ class Wordpress():
) )
return final_dict return final_dict
def post_events(self, events): def get_nonce(self):
metadata = self.calendar_meadata.to_dict() r = self.session.get(
data = self._generate_data(events) f'{self.base_url}/wp-admin/admin.php?page=wpbs-calendars&subpage=edit-calendar&calendar_id=1',
)
soup = BeautifulSoup(r.text, 'html.parser')
nonce = soup.find_all('input', {'id': 'wpbs_token'})[0]
return nonce['value']
def post_events(self, events, start: datetime.datetime | None=None, until: datetime.timedelta | None=None):
metadata = self.calendar_metadata.to_dict()
data = self._generate_data(events, start=start, until=until)
update_request = self.session.post( update_request = self.session.post(
f'{self.base_url}/wp-admin/admin-ajax.php', f'{self.base_url}/wp-admin/admin-ajax.php',
auth=(self.credentials['user'], self.credentials['password']), auth=(self.credentials['user'], self.credentials['password']),
@ -141,7 +170,11 @@ class Wordpress():
'action': 'wpbs_save_calendar_data', 'action': 'wpbs_save_calendar_data',
'form_data': urllib.parse.urlencode(metadata), 'form_data': urllib.parse.urlencode(metadata),
'calendar_data': json.dumps(data), 'calendar_data': json.dumps(data),
'wpbs_token': self.get_nonce(),
}, },
) )
return update_request r = 'wpbs_message=calendar_update_success' in update_request.text
if not r:
raise Exception(f'failed to post events, got answer {update_request.text}')

41
ci/pipeline.yaml Normal file
View file

@ -0,0 +1,41 @@
---
resources:
- name: periodic
type: time
source:
interval: 24h
- name: script
type: git
source:
uri: https://gitea.redxef.at/redxef/wp-cal-integration
branch: master
jobs:
- name: update-calendar
plan:
- get: periodic
trigger: true
- get: script
- task: run-update
config:
platform: linux
image_resource:
type: registry-image
source:
repository: alpine
tag: latest
inputs:
- name: script
path: .
params:
configuration_json: ((configuration))
run:
path: sh
args:
- -c
- |
#!/usr/bin/env sh
apk --no-cache --update add libcap jq python3 py3-pip
python3 -m pip install --break-system-packages --requirement requirements.txt
echo "$configuration_json" | ./main.py -l :verbose -c -

230
main.py
View file

@ -1,67 +1,197 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import os
import sys import sys
import logging
import datetime
import yaml import yaml
try: from schema import Schema, And, Or, Use, Optional, Regex
from yaml import CLoader as Loader
except ImportError:
from yaml import Loader
import click
import adapters
from adapters import * from adapters import *
class Config(): logging.VERBOSE = (logging.DEBUG + logging.INFO) // 2
""" logger = logging.getLogger(f'wp_cal')
The default configuration.
def range_str_to_timedelta(value):
valid_units = 'smhdw'
current_int = 0
values = {}
for c in value:
if c in '0123456789':
current_int *= 10
current_int += int(c)
elif c in valid_units:
if c in values:
logger.warning('unit %s already in values, overwriting', c)
values[c] = current_int
current_int = 0
c = 's'
if current_int != 0:
if c in values:
logger.warning('unit %s already in values, overwriting', c)
values['s'] = current_int
for valid_unit in valid_units:
values.setdefault(valid_unit, 0)
return datetime.timedelta(
seconds=values['s'],
minutes=values['m'],
hours=values['h'],
days=values['d'],
weeks=values['w']
)
Keys: def multiple_timedeltas(value):
.google.calendar_id: the id of the calendar to sync mappings = {}
.google.credentials: the json of the obtained credentials file for v in value.split(','):
.google.token_file: where to save the login token kv = v.split(':')
.wordpress.url: the base wordpress url if len(kv) == 1:
.wordpress.calendar.id: the id of the (wp-booking-system) calendar mappings[-1] = range_str_to_timedelta(kv[0])
.wordpress.calendar.name: the name of the calendar continue
.wordpress.calendar.translations: a dictionary of language <-> translation pairs (example: {"en": "Reservations"}) mappings[int(kv[0])] = range_str_to_timedelta(kv[1])
.wordpress.credentials.user: the user as which to log into wordpress return mappings
.wordpress.credentials.password: the users password
"""
def __init__(self, file): config_schema = Schema({
self.file = file 'sources': [dict],
self.config: dict | None = None 'sinks': [dict],
Optional('logging', default={'level': ':WARNING,wp_cal:INFO'}): {
Optional('level', default=':WARNING,wp_cal:INFO'): Use(str),
},
Optional('range', default='365d'): Or(
dict,
And(
Use(str),
Regex(r'(([0-9]+:)?[0-9]+[smhdw],?)*'),
Use(multiple_timedeltas),
),
),
})
def load(self): def load_config(file):
if self.file == '-': if file == '-':
config = yaml.load(sys.stdin, Loader=Loader) config = yaml.safe_load(sys.stdin)
else: else:
with open(self.file) as fp: if os.stat(file).st_mode & 0o777 & ~0o600:
config = yaml.load(fp, Loader=Loader) raise Exception('refusing to load insecure configuration file, file must have permission 0o600')
self.config = config with open(file) as fp:
config = yaml.safe_load(fp)
return config_schema.validate(config)
def __getitem__(self, name): def init_logging():
assert self.config is not None logging.getLogger().addHandler(
return self.config[name] logging.StreamHandler(),
)
def set_logging_level(level: str):
levels = {
'NOTSET': logging.NOTSET,
'DEBUG': logging.DEBUG,
'VERBOSE': logging.VERBOSE,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'WARN': logging.WARNING,
'ERROR': logging.ERROR,
'CRITICAL': logging.CRITICAL,
}
log_levels = [x.split(':') for x in level.split(',')]
for module, level in log_levels:
module = module.strip() if module else None
level = level.strip().upper()
if level not in levels:
raise ValueError(f'invalid log level, allowed values: {repr(set(levels.keys()))}')
logging.getLogger(module).setLevel(levels[level])
@click.command()
@click.option('--config', '-c', envvar='WP_CAL_CONFIG', default='-', help='The configuration file')
@click.option('--dryrun', '-d', envvar='WP_CAL_DRYRUN', is_flag=True, help="Don't actually post any data, just show it")
@click.option('--level', '-l', envvar='WP_CAL_LEVEL', default=None, help='The log level for the application')
@click.option('--range', '-r', envvar='WP_CAL_RANGE', default=None, help='The time range from start to start + range to synchronize events')
def main(config, dryrun, level, range):
init_logging()
set_logging_level(':DEBUG')
config = load_config(config)
if level:
config['logging']['level'] = level
if range:
config['range'] = range
config = config_schema.validate(config)
set_logging_level(config['logging']['level'])
sources = set()
sinks = set()
for source in config['sources']:
assert len(source.keys()) == 1
for aname, a in adapters.ADAPTERS.items():
if aname in source:
sources |= {a.new(source[aname])}
break
else:
logger.error('couldn\'t find valid adapter for source configuration %s', source)
sys.exit(1)
for sink in config['sinks']:
assert len(sink.keys()) == 1
for aname, a in adapters.ADAPTERS.items():
if aname in sink:
sinks |= {a.new(sink[aname])}
break
else:
logger.error('couldn\'t find valid adapter for sink configuration %s', sink)
sys.exit(1)
if not all([isinstance(x, adapters.Source) for x in sources]):
logger.error('one or more source configurations do not implement being a source')
sys.exit(1)
if not all([isinstance(x, adapters.Sink) for x in sinks]):
logger.error('one or more sink configurations do not implement being a sink')
sys.exit(1)
# log in
if not all([x.login() for x in sources | sinks]):
logger.error('failed to log into one or more sinks or sources')
sys.exit(1)
# gather events
events = []
source_results = []
for i, source in enumerate(sources):
try:
events += source.get_events_resolved(until=config['range'].get(i, config['range'][-1]))
source_results += [True]
except Exception:
logger.exception('failed to get events from source %s', source)
source_results += [False]
if not any(source_results):
logger.error('event get failed for all sources')
sys.exit(1)
# filter cancelled events
logger.info('found %d events', len(events))
logger.info('not syncing cancelled events')
events = [e for e in events if e['status'] != 'cancelled']
logger.info('syncing %d events', len(events))
logger.log(logging.VERBOSE, 'events are: %s', [f"{e.get('summary', '<NO SUMMARY PROVIDED>')} ({e.get('start')})" for e in events])
# post events
if not dryrun:
sink_results = []
for sink in sinks:
try:
sink.post_events(events, until=max(config['range'].values()))
sink_results += [True]
except Exception:
logger.exception('failed to post to sink %s', sink)
sink_results += [False]
if not any(sink_results):
logger.error('event post failed for all sinks')
sys.exit(1)
logger.info("done")
sys.exit(0)
if __name__ == '__main__': if __name__ == '__main__':
config = Config('-') main()
config.load()
g = Google(
config['google']['calendar_id'],
credentials=config['google']['credentials'],
token_file=config['google'].get('token_file', '~/.wp-cal-integration-google-token.json')
)
w = Wordpress(
config['wordpress']['url'],
calendar_metadata=CalendarMetadata(
id=config['wordpress']['calendar']['id'],
name=config['wordpress']['calendar']['name'],
translations=config['wordpress']['calendar']['translations'],
),
credentials=config['wordpress']['credentials'],
)
g.login()
events = g.get_events()
w.login()
w.post_events(events)

8
requirements.txt Normal file
View file

@ -0,0 +1,8 @@
pyyaml
schema
google-api-python-client
google-auth-httplib2
google-auth-oauthlib
click
bs4
python-dateutil