Update only every day.

Alpine edge failed with python3 no module pip, after py3-pip
being installed.

.gitignore

@@ -1 +1,4 @@
 __pycache__
+wp-cal-integration
+*.c
+*.o

Dockerfile

@@ -0,0 +1,12 @@
+FROM alpine
+
+RUN mkdir -p /opt/wp-cal-integration \
+ && apk --no-cache add python3 py3-pip
+COPY main.py /opt/wp-cal-integration/
+COPY adapters /opt/wp-cal-integration/adapters
+COPY requirements.txt /opt/wp-cal-integration/
+RUN python3 -m pip install -r /opt/wp-cal-integration/requirements.txt
+VOLUME /etc/wp-cal-integration/
+
+ENTRYPOINT [ "/opt/wp-cal-integration/main.py" ]
+CMD [ "-c", "/etc/wp-cal-integration/config.yaml" ]

Makefile

@@ -0,0 +1,21 @@
+CC ?= cc
+CFLAGS ?= $(shell python-config --embed --cflags)
+LDFLAGS ?= $(shell python-config --embed --ldflags)
+
+SRC := main.py adapters/*.py
+OBJ := $(SRC:%.py=%.o)
+
+wp-cal-integration: $(OBJ)
+	$(CC) $(LDFLAGS) -o $@ $^
+
+clean:
+	$(RM) $(OBJ)
+
+main.c: main.py
+	cython -3 --embed -o $@ $<
+%.c: %.py
+	cython -3 -o $@ $<
+
+%.o: %.c
+	$(CC) -c $(CFLAGS) -o $@ $<
+

README.md

@@ -1,5 +1,17 @@
 # Wordpress WP Booking System/Google Calendar integration
 
+## Usage
+
+1. Create a google service account which has permission to read the desired calendars (add it to the calendars).
+2. Add a key to the service account (choose the json option) and save it somewhere secure.
+3. Run this program once `./main.py --config config.yaml` it will generate a example configuration file.
+4. Add the json contents into the example file under `.google.credentials` and fill in the rest of the options.
+5. To synchronize once run `./main.py --config config.yaml`.
+
+## CI
+
+There is a concourse pipeline which runs once every hour to synchronize all events. It reads the configuration
+file from the concourse vault and passes it to the script.
 
 ## Known issues
 

adapters/__init__.py

@@ -1,11 +1,28 @@
-
-
 from .google import Google
 from .wordpress import Wordpress
-from .wordpress import CalendarMetadata
+from .abc import Source, Sink, Adapter
+
+_ADAPTERS: set[type[Adapter]] = {
+    Google,
+    Wordpress,
+}
+_SOURCES: set[type[Source]] = {
+    Google,
+}
+_SINKS: set[type[Sink]] = {
+    Wordpress,
+}
+ADAPTERS: dict[str, type[Adapter]] = {
+    str(x.__name__).lower(): x for x in _ADAPTERS
+}
+SOURCES: dict[str, type[Source]] = {
+    str(x.__name__).lower(): x for x in _SOURCES
+}
+SINKS: dict[str, type[Sink]] = {
+    str(x.__name__).lower(): x for x in _SINKS
+}
 
 __all__ = [
     'Google',
     'Wordpress',
-    'CalendarMetadata',
 ]

adapters/abc.py

@@ -0,0 +1,85 @@
+import abc
+import typing
+import datetime
+import copy
+
+import schema
+import dateutil.rrule
+
+class Adapter(abc.ABC):
+    schema: schema.Schema
+
+    def __init__(self, config, *args, **kwargs):
+        _ = args, kwargs
+        self.config = config
+
+    @classmethod
+    def new(cls, config: dict, *args, **kwargs):
+        return cls(cls.schema.validate(config) , *args, **kwargs)
+
+    @abc.abstractmethod
+    def login(self) -> bool:
+        raise NotImplementedError()
+
+class Source(abc.ABC):
+
+    @abc.abstractmethod
+    def get_events(
+        self,
+        start: datetime.datetime | None=None,
+        until: datetime.timedelta | None=None,
+        limit=None,
+    ) -> typing.Iterable[dict]:
+        raise NotImplementedError()
+
+    def get_events_resolved(
+        self,
+        start: datetime.datetime | None=None,
+        until: datetime.timedelta | None=None,
+        limit=None,
+    ) -> typing.Iterable[dict]:
+        until = until if until else datetime.timedelta(days=365)
+        now = start if start else datetime.datetime.utcnow().astimezone()
+        now_365 = now + until
+        for e in self.get_events(
+            start=start,
+            until=until,
+            limit=limit,
+        ):
+            if 'recurrence' not in e:
+                yield e
+                continue
+            r = e.pop('recurrence')
+            r = dateutil.rrule.rrulestr(
+                '\n'.join(r),
+                unfold=True,
+                ignoretz=True,
+                dtstart=datetime.datetime.fromisoformat(
+                    e['start']['dateTime']
+                        if 'dateTime' in e['start'] else
+                    e['start']['date']
+                ).replace(tzinfo=None)
+            )
+            for t_ in r.between(now.replace(tzinfo=None), now_365.replace(tzinfo=None)):
+                e_ = copy.deepcopy(e)
+                if 'dateTime' in e['start']:
+                    e_['start']['dateTime'] = datetime.datetime.combine(t_.date(), datetime.datetime.fromisoformat(e['start']['dateTime']).time()).isoformat()
+                elif 'date' in e['start']:
+                    e_['start']['date'] = datetime.datetime.combine(t_.date(), datetime.time())
+                if 'dateTime' in e['end']:
+                    e_['end']['dateTime'] = datetime.datetime.combine(t_.date(), datetime.datetime.fromisoformat(e['end']['dateTime']).time()).isoformat()
+                elif 'date' in e['end']:
+                    e_['end']['date'] = datetime.datetime.combine(t_.date(), datetime.time())
+                yield e_
+            yield e
+
+class Sink(abc.ABC):
+
+    @abc.abstractmethod
+    def post_events(
+        self,
+        events,
+        start: datetime.datetime | None=None,
+        until: datetime.timedelta | None=None,
+    ) -> bool:
+        raise NotImplementedError()

adapters/google.py

@@ -1,49 +1,55 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import tempfile
 import datetime
-import os.path
+import logging
+import os
 
-import json
 from google.auth.transport.requests import Request
-from google.oauth2.credentials import Credentials
+#from google.oauth2.credentials import Credentials
+from google.oauth2.service_account import Credentials
 from google_auth_oauthlib.flow import InstalledAppFlow
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
+from schema import Use, Optional, Schema
+
+logger = logging.getLogger(f"wp_cal.{__name__}")
+
+from .abc import Source, Adapter
 
 # If modifying these scopes, delete the file token.json.
-SCOPES = ['https://www.googleapis.com/auth/calendar']
+SCOPES = ['https://www.googleapis.com/auth/calendar.readonly']
 
 
-class Google():
+class Google(Source, Adapter):
 
-    def __init__(self, calendar_id, *, credentials, token_file):
-        _, credentials_file = tempfile.mkstemp()
-        with open(credentials_file, 'w') as fp:
-            json.dump(credentials, fp)
+    schema = Schema({
+        'calendar_id': Use(str),
+        'credentials': Use(dict),
+        Optional(
+            'token_file',
+            default=os.path.join(
+                os.environ["HOME"],
+                '.wp-cal-google-token',
+            ),
+        ): Use(str),
+    })
 
-        self.calendar_id = calendar_id
-        self.credentials_file = credentials_file
-        self.token_file = token_file
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.calendar_id = self.config['calendar_id']
+        self.credentials_info = self.config['credentials']
+        self.token_file = self.config['token_file']
+
+        # runtime data
         self.credentials = None
 
-    def login(self):
-        if os.path.exists(self.token_file):
-            self.credentials = Credentials.from_authorized_user_file(self.token_file, SCOPES)
-        if self.credentials is None or not self.credentials.valid:
-            if self.credentials is not None \
-                    and self.credentials.expired \
-                    and  self.credentials.refresh_token:
-                self.credentials.refresh(Request())
-            else:
-                flow = InstalledAppFlow.from_client_secrets_file(
-                    self.credentials_file,
-                    SCOPES,
-                )
-                self.credentials = flow.run_local_server(port=0)
-            with open(self.token_file, 'w') as token:
-                token.write(self.credentials.to_json())
+    def login(self) -> bool:
+        self.credentials = Credentials.from_service_account_info(self.credentials_info, scopes=SCOPES)
+        if not self.credentials.valid:
+            self.credentials.refresh(Request())
+        logger.debug('credentials.valid = %s', repr(self.credentials.valid))
+        return self.credentials.valid
 
     def get_events(self, start: datetime.datetime | None=None, until: datetime.timedelta | None=None, limit=None):
         until = until if until else datetime.timedelta(days=365)
@@ -59,7 +65,7 @@ class Google():
                 maxResults=limit,
             ).execute()
         except HttpError:
-            print('an error occured')
+            logger.exception("a http error occured")
             raise
         events = events.get('items', [])
         return events

adapters/utils.py

@@ -1,6 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import logging
+
+logger = logging.getLogger(f"wp_cal.{__name__}")
+
 def dict_merge(d0, d1):
     for k, v in d1.items():
         if (k in d0 and isinstance(d0[k], dict) and isinstance(d1[k], dict)):

adapters/wordpress.py

@@ -5,10 +5,15 @@ import requests
 import urllib.parse
 import json
 import datetime
+import logging
+
+from schema import Use, Schema
+from bs4 import BeautifulSoup
 
 from . import utils
+from .abc import Sink, Adapter
 
-
+logger = logging.getLogger(f"wp_cal.{__name__}")
 
 class CalendarMetadata():
 
@@ -32,19 +37,29 @@ class CalendarMetadata():
             **translations,
         }
 
-class Wordpress():
+class Wordpress(Sink, Adapter):
 
-    def __init__(
-        self,
-        base_url: str,
-        *,
-        calendar_metadata: CalendarMetadata,
-        credentials: dict,
-    ):
-        self.base_url = base_url
+    schema = Schema({
+        'url': Use(str),
+        'calendar': {
+            'id': Use(str),
+            'name': Use(str),
+            'translations': Use(dict),
+        },
+        'credentials': {
+            'user': Use(str),
+            'password': Use(str),
+        },
+    })
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.base_url = self.config['url']
+        self.credentials = self.config['credentials']
+        self.calendar_metadata = CalendarMetadata(**self.config['calendar'])
+
+        # runtime data
         self.session = requests.Session()
-        self.credentials = credentials
-        self.calendar_meadata = calendar_metadata
 
     def login(self):
         login_request = self.session.post(
@@ -55,9 +70,15 @@ class Wordpress():
                 'wp-submit': 'Anmelden',
                 'redirect_to': f'{self.base_url}/wp-admin/',
                 'testcookie': 1,
-            }
+            },
+            allow_redirects=False,
         )
-        return login_request
+        logger.debug('login request return headers = %s', login_request.headers)
+        login_request_cookies = [x.strip() for x in login_request.headers['Set-Cookie'].split(',')]
+        if len(login_request_cookies) > 1:
+            logger.debug('login seems to be ok, cookies = %s', login_request_cookies)
+            return True
+        return False
 
     def _datestr_to_date(self, s, tz=None):
         if s.endswith('Z'):
@@ -117,8 +138,8 @@ class Wordpress():
             start: datetime.datetime | None=None,
             until: datetime.timedelta | None=None
     ):
-        until = until if until else datetime.timedelta(days=365)
         start = start if start else datetime.datetime.utcnow().astimezone()
+        until = until if until else datetime.timedelta(days=365)
 
         final_dict = {}
         for event in events:
@@ -131,9 +152,17 @@ class Wordpress():
         )
         return final_dict
 
-    def post_events(self, events):
-        metadata = self.calendar_meadata.to_dict()
-        data = self._generate_data(events)
+    def get_nonce(self):
+        r = self.session.get(
+            f'{self.base_url}/wp-admin/admin.php?page=wpbs-calendars&subpage=edit-calendar&calendar_id=1',
+        )
+        soup = BeautifulSoup(r.text, 'html.parser')
+        nonce = soup.find_all('input', {'id': 'wpbs_token'})[0]
+        return nonce['value']
+
+    def post_events(self, events, start: datetime.datetime | None=None, until: datetime.timedelta | None=None):
+        metadata = self.calendar_metadata.to_dict()
+        data = self._generate_data(events, start=start, until=until)
         update_request = self.session.post(
             f'{self.base_url}/wp-admin/admin-ajax.php',
             auth=(self.credentials['user'], self.credentials['password']),
@@ -141,7 +170,11 @@ class Wordpress():
                 'action': 'wpbs_save_calendar_data',
                 'form_data': urllib.parse.urlencode(metadata),
                 'calendar_data': json.dumps(data),
+                'wpbs_token': self.get_nonce(),
             },
         )
-        return update_request
+        r = 'wpbs_message=calendar_update_success' in update_request.text
+        if not r:
+            raise Exception(f'failed to post events, got answer {update_request.text}')
+
 

ci/pipeline.yaml

@@ -0,0 +1,41 @@
+---
+resources:
+- name: periodic
+  type: time
+  source:
+    interval: 24h
+
+- name: script
+  type: git
+  source:
+    uri: https://gitea.redxef.at/redxef/wp-cal-integration
+    branch: master
+
+jobs:
+- name: update-calendar
+  plan:
+  - get: periodic
+    trigger: true
+  - get: script
+  - task: run-update
+    config:
+      platform: linux
+      image_resource:
+        type: registry-image
+        source:
+          repository: alpine
+          tag: latest
+      inputs:
+      - name: script
+        path: .
+      params:
+        configuration_json: ((configuration))
+      run:
+        path: sh
+        args:
+        - -c
+        - |
+          #!/usr/bin/env sh
+          apk --no-cache --update add libcap jq python3 py3-pip
+          python3 -m pip install --break-system-packages --requirement requirements.txt
+          echo "$configuration_json" | ./main.py -l :verbose -c -

main.py

@@ -1,67 +1,197 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import os
 import sys
+import logging
+import datetime
 import yaml
-try:
-    from yaml import CLoader as Loader
-except ImportError:
-    from yaml import Loader
+from schema import Schema, And, Or, Use, Optional, Regex
 
+import click
+
+import adapters
 from adapters import *
 
-class Config():
+logging.VERBOSE = (logging.DEBUG + logging.INFO) // 2
 
-    """
-    The default configuration.
+logger = logging.getLogger(f'wp_cal')
 
+def range_str_to_timedelta(value):
+    valid_units = 'smhdw'
+    current_int = 0
+    values = {}
+    for c in value:
+        if c in '0123456789':
+            current_int *= 10
+            current_int += int(c)
+        elif c in valid_units:
+            if c in values:
+                logger.warning('unit %s already in values, overwriting', c)
+            values[c] = current_int
+            current_int = 0
+    c = 's'
+    if current_int != 0:
+        if c in values:
+            logger.warning('unit %s already in values, overwriting', c)
+        values['s'] = current_int
+    for valid_unit in valid_units:
+        values.setdefault(valid_unit, 0)
+    return datetime.timedelta(
+        seconds=values['s'],
+        minutes=values['m'],
+        hours=values['h'],
+        days=values['d'],
+        weeks=values['w']
+    )
 
-    Keys:
-    .google.calendar_id: the id of the calendar to sync
-    .google.credentials: the json of the obtained credentials file
-    .google.token_file: where to save the login token
-    .wordpress.url: the base wordpress url
-    .wordpress.calendar.id: the id of the (wp-booking-system) calendar
-    .wordpress.calendar.name: the name of the calendar
-    .wordpress.calendar.translations: a dictionary of language <-> translation pairs (example: {"en": "Reservations"})
-    .wordpress.credentials.user: the user as which to log into wordpress
-    .wordpress.credentials.password: the users password
-    """
+def multiple_timedeltas(value):
+    mappings = {}
+    for v in value.split(','):
+        kv = v.split(':')
+        if len(kv) == 1:
+            mappings[-1] = range_str_to_timedelta(kv[0])
+            continue
+        mappings[int(kv[0])] = range_str_to_timedelta(kv[1])
+    return mappings
 
-    def __init__(self, file):
-        self.file = file
-        self.config: dict | None = None
+config_schema = Schema({
+    'sources': [dict],
+    'sinks': [dict],
+    Optional('logging', default={'level': ':WARNING,wp_cal:INFO'}): {
+        Optional('level', default=':WARNING,wp_cal:INFO'): Use(str),
+    },
+    Optional('range', default='365d'): Or(
+        dict,
+        And(
+            Use(str),
+            Regex(r'(([0-9]+:)?[0-9]+[smhdw],?)*'),
+            Use(multiple_timedeltas),
+        ),
+    ),
+})
 
-    def load(self):
-        if self.file == '-':
-            config = yaml.load(sys.stdin, Loader=Loader)
+def load_config(file):
+    if file == '-':
+        config = yaml.safe_load(sys.stdin)
+    else:
+        if os.stat(file).st_mode & 0o777 & ~0o600:
+            raise Exception('refusing to load insecure configuration file, file must have permission 0o600')
+        with open(file) as fp:
+            config = yaml.safe_load(fp)
+    return config_schema.validate(config)
+
+def init_logging():
+    logging.getLogger().addHandler(
+        logging.StreamHandler(),
+    )
+
+def set_logging_level(level: str):
+    levels = {
+        'NOTSET': logging.NOTSET,
+        'DEBUG': logging.DEBUG,
+        'VERBOSE': logging.VERBOSE,
+        'INFO': logging.INFO,
+        'WARNING': logging.WARNING,
+        'WARN': logging.WARNING,
+        'ERROR': logging.ERROR,
+        'CRITICAL': logging.CRITICAL,
+    }
+
+    log_levels = [x.split(':') for x in level.split(',')]
+    for module, level in log_levels:
+        module = module.strip() if module else None
+        level = level.strip().upper()
+        if level not in levels:
+            raise ValueError(f'invalid log level, allowed values: {repr(set(levels.keys()))}')
+        logging.getLogger(module).setLevel(levels[level])
+
+@click.command()
+@click.option('--config', '-c', envvar='WP_CAL_CONFIG', default='-', help='The configuration file')
+@click.option('--dryrun', '-d', envvar='WP_CAL_DRYRUN', is_flag=True, help="Don't actually post any data, just show it")
+@click.option('--level', '-l', envvar='WP_CAL_LEVEL', default=None, help='The log level for the application')
+@click.option('--range', '-r', envvar='WP_CAL_RANGE', default=None, help='The time range from start to start + range to synchronize events')
+def main(config, dryrun, level, range):
+    init_logging()
+    set_logging_level(':DEBUG')
+
+    config = load_config(config)
+    if level:
+        config['logging']['level'] = level
+    if range:
+        config['range'] = range
+    config = config_schema.validate(config)
+    set_logging_level(config['logging']['level'])
+
+    sources = set()
+    sinks = set()
+    for source in config['sources']:
+        assert len(source.keys()) == 1
+        for aname, a in adapters.ADAPTERS.items():
+            if aname in source:
+                sources |= {a.new(source[aname])}
+                break
         else:
-            with open(self.file) as fp:
-                config = yaml.load(fp, Loader=Loader)
-        self.config = config
+            logger.error('couldn\'t find valid adapter for source configuration %s', source)
+            sys.exit(1)
+    for sink in config['sinks']:
+        assert len(sink.keys()) == 1
+        for aname, a in adapters.ADAPTERS.items():
+            if aname in sink:
+                sinks |= {a.new(sink[aname])}
+                break
+        else:
+            logger.error('couldn\'t find valid adapter for sink configuration %s', sink)
+            sys.exit(1)
 
-    def __getitem__(self, name):
-        assert self.config is not None
-        return self.config[name]
+    if not all([isinstance(x, adapters.Source) for x in sources]):
+        logger.error('one or more source configurations do not implement being a source')
+        sys.exit(1)
+    if not all([isinstance(x, adapters.Sink) for x in sinks]):
+        logger.error('one or more sink configurations do not implement being a sink')
+        sys.exit(1)
+
+    # log in
+    if not all([x.login() for x in sources | sinks]):
+        logger.error('failed to log into one or more sinks or sources')
+        sys.exit(1)
+
+    # gather events
+    events = []
+    source_results = []
+    for i, source in enumerate(sources):
+        try:
+            events += source.get_events_resolved(until=config['range'].get(i, config['range'][-1]))
+            source_results += [True]
+        except Exception:
+            logger.exception('failed to get events from source %s', source)
+            source_results += [False]
+    if not any(source_results):
+        logger.error('event get failed for all sources')
+        sys.exit(1)
+    # filter cancelled events
+    logger.info('found %d events', len(events))
+    logger.info('not syncing cancelled events')
+    events = [e for e in events if e['status'] != 'cancelled']
+    logger.info('syncing %d events', len(events))
+    logger.log(logging.VERBOSE, 'events are: %s', [f"{e.get('summary', '<NO SUMMARY PROVIDED>')} ({e.get('start')})" for e in events])
+
+    # post events
+    if not dryrun:
+        sink_results = []
+        for sink in sinks:
+            try:
+                sink.post_events(events, until=max(config['range'].values()))
+                sink_results += [True]
+            except Exception:
+                logger.exception('failed to post to sink %s', sink)
+                sink_results += [False]
+        if not any(sink_results):
+            logger.error('event post failed for all sinks')
+            sys.exit(1)
+
+    logger.info("done")
+    sys.exit(0)
 
 if __name__ == '__main__':
-    config = Config('-')
-    config.load()
-    g = Google(
-        config['google']['calendar_id'],
-        credentials=config['google']['credentials'],
-        token_file=config['google'].get('token_file', '~/.wp-cal-integration-google-token.json')
-    )
-    w = Wordpress(
-        config['wordpress']['url'],
-        calendar_metadata=CalendarMetadata(
-            id=config['wordpress']['calendar']['id'],
-            name=config['wordpress']['calendar']['name'],
-            translations=config['wordpress']['calendar']['translations'],
-        ),
-        credentials=config['wordpress']['credentials'],
-    )
-    g.login()
-    events = g.get_events()
-    w.login()
-    w.post_events(events)
+    main()

requirements.txt

@@ -0,0 +1,8 @@
+pyyaml
+schema
+google-api-python-client
+google-auth-httplib2
+google-auth-oauthlib
+click
+bs4
+python-dateutil