Add Google Calendar sync that allows sync from calendars that a user has access to, even if they might not have an iCal available.

.gitignore

@@ -218,5 +218,6 @@ venv/
 static/
 
 davinci/local.py
+credentials.json
 
 *.sqlite3

README.md

@@ -1,5 +1,6 @@
 # DAVinci
-DAVinci is a Django application that can keep remote iCalendar files (.ics) synchronized with a CalDAV calendar. It allows for multiple remote ICS files being synchronized to multiple calendars.
+DAVinci is a Django application that can keep remote iCalendar files (.ics) or Google Calendars synchronized with 
+a CalDAV calendar. It allows for multiple remote ICS files being synchronized to multiple calendars.
 
 ## Requirements
 - Python 3.8
@@ -48,8 +49,9 @@ When deploying, be sure to set the `STATIC_ROOT` setting, run `python manage.py
 serve the static files directory under `/static` on the web server.
 
 ### Configuring auto-sync
-To enable the synchronisation, please configure the following command to be executed every minute (to allow update intervals down to 1 minute). 
-You can run the command every hour as well, but then the minimum update interval will be 1 hour.
+To enable the synchronisation, please configure the following command to be executed every 10 minutes 
+(to allow update intervals down to 10 minutes). You can run the command less frequently as well, 
+but then the minimum update interval will be limited to that interval.
 
 ```bash
 python manage.py ical_sync
@@ -59,7 +61,7 @@ The configuration can be done for example by using cron:
 
 ```
 # Run ical sync every minute
-* * * * * /path/to/python /path/to/manage.py ical_sync
+*/10 * * * * /path/to/python /path/to/manage.py ical_sync
 ```
 
 Or via a SystemD timer:
@@ -89,14 +91,30 @@ Requires=davinci_ical_syc.service
 [Timer]
 Unit=davinci_ical_syc.service
 OnBootSec=5min
-OnCalendar=*:0/1
+OnCalendar=*:0/10
 
 [Install]
 WantedBy=timers.target
 ``` 
 
+### Configuring Google Calendar sync
+Google Calendar sync makes use of the Google [CalDAV API](https://developers.google.com/calendar/caldav/v2/guide). 
+Because of this, it requires a valid Google API application which is configured to access this API.
+
+In the [Google API console](https://console.developers.google.com/project), create a project. Then activate the 
+*CalDAV API*, and create OAuth 2.0 credentials. Download the credential JSON file and put it in the main directory of
+this application (same directory as `manage.py`), name it `credentials.json`.
+
+If you are asked for the required scopes, add (at least) the following:
+- `https://www.googleapis.com/auth/userinfo.email`
+- `https://www.googleapis.com/auth/calendar.readonly`
+- `https://www.googleapis.com/auth/calendar.events.readonly`
+
 ## Usage
 Visit `/admin` and login with your created admin account to manage the synchronisations.
 
-First, add a CalDAV server. Then, add the CalDAV Calendars that you want to sync to, and lastly, create iCalSync objects for each remote `.ics` file.
+First, add a CalDAV server. Then, add the CalDAV Calendars that you want to sync to, and lastly, create iCalSync 
+objects for each remote `.ics` file.
 
+For Google Calendar Sync, first go to "Google Authentication Information" and authorize a Google account. Then, 
+add the gCalSync objects under "Google Calendar Syncs".

davinci/caldav/models.py

@@ -65,13 +65,15 @@ class CalDAVCalendar(models.Model):
                 return cal
         return None
 
-    def upload_events(self, events: List[Event], purge: bool = True) -> Tuple[int, int, int]:
+    def upload_events(self, events: List[Event], purge: bool = True, ignore_invalid_recurring: bool = False) -> Tuple[int, int, int]:
         """
         Upload events to this calendar. If purge is True, all old events will be removed.
         :param events: The events to upload
         :type events: List[Event]
         :param purge: If old events in the calendar should be purged or not.
         :type purge: bool
+        :param ignore_invalid_recurring: Whether to ignore errors when adding invalid recurring events (GCalendar does this)
+        :type ignore_invalid_recurring: bool
         :return Amount of events added, updated, and deleted
         :rtype Tuple[int, int, int]
         """
@@ -82,9 +84,7 @@ class CalDAVCalendar(models.Model):
             raise ValueError(f"No calendar found on url '{self.calendar_url}'")
 
         # Keep track of which events were already in the calendar
-        purge_uids = []
-        if purge:
-            purge_uids = [e.instance.vevent.uid.value for e in calendar.events()]
+        purge_uids = [e.instance.vevent.uid.value for e in calendar.events()]
 
         updated = 0
         added = 0
@@ -105,13 +105,16 @@ class CalDAVCalendar(models.Model):
                     cevent.data = vcal.fix(ev)
                     cevent.save()
                 except Exception as e:
-                    logger.error(f"Error during saving of event {event.uid}: {e}")
-                    logger.error(f"======EVENT======\n{ev}\n=================")
+                    if ignore_invalid_recurring and len([x for x in event.extra if x.name == "RECURRENCE-ID"]) > 0:
+                        # Recurring event from GCal, ignore
+                        pass
+                    else:
+                        logger.error(f"Error during saving of event {event.uid}: {e}")
+                        logger.error(f"======EVENT======\n{ev}\n=================")
 
                 purge_uids.remove(event.uid)
                 updated += 1
 
-
             else:
                 try:
                     calendar.save_event(ev)
@@ -121,8 +124,11 @@ class CalDAVCalendar(models.Model):
                 added += 1
 
         # Purge old events
-        deleted = len(purge_uids)
-        for uid in purge_uids:
-            calendar.event_by_uid(uid).delete()
+        if purge:
+            deleted = len(purge_uids)
+            for uid in purge_uids:
+                calendar.event_by_uid(uid).delete()
+        else:
+            deleted = 0
 
         return added, updated, deleted

davinci/gcalendar/admin.py

@@ -0,0 +1,21 @@
+from django.contrib import admin
+from davinci.gcalendar.models import GoogleAuthInfo
+from davinci.gcalendar.forms import GCalSyncForm
+from davinci.gcalendar.models import GCalSync
+
+
+class GoogleAuthInfoAdmin(admin.ModelAdmin):
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+
+class GCalSyncAdmin(admin.ModelAdmin):
+    form = GCalSyncForm
+    list_display = ['name', 'target', 'purge', 'humanized_sync_interval', 'last_sync', 'active']
+
+
+admin.site.register(GoogleAuthInfo, GoogleAuthInfoAdmin)
+admin.site.register(GCalSync, GCalSyncAdmin)

davinci/gcalendar/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class GcalendarConfig(AppConfig):
+    name = 'davinci.gcalendar'

davinci/gcalendar/forms.py

@@ -0,0 +1,14 @@
+from django import forms
+
+from davinci.gcalendar.models import GCalSync
+
+from durationwidget.widgets import TimeDurationWidget
+
+
+class GCalSyncForm(forms.ModelForm):
+    class Meta:
+        model = GCalSync
+        widgets = {
+            'sync_interval': TimeDurationWidget(show_seconds=False)
+        }
+        fields = "__all__"

davinci/gcalendar/management/commands/gcal_sync.py

@@ -0,0 +1,30 @@
+import logging
+
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from davinci.gcalendar.models import GCalSync
+from davinci.icalendar.decorators import handle_lock
+
+
+class Command(BaseCommand):
+    help = 'Sychronize Google Calendar syncs that need to be synchronized.'
+
+    @handle_lock
+    def handle(self, *args, **options):
+        logger = logging.getLogger("davinci.gcalendar.gcal_sync")
+        logger.debug(f"Command `gcal_sync` invoked")
+
+        for sync in GCalSync.objects.filter(active=True):
+            sync: GCalSync
+            if sync.last_sync is None or timezone.now() > (sync.last_sync + sync.sync_interval):
+                logger.debug(f"Synchronizing {sync.name}...")
+
+                events = sync.get_events()
+                added, existing, deleted = sync.target.upload_events(events, purge=sync.purge, ignore_invalid_recurring=True)
+
+                sync.last_sync = timezone.now()
+                sync.save()
+                logger.debug(f"Sync for {sync.name} done! {added} events added, {existing} updated, and {deleted} purged.")
+
+        logger.debug("iCal sync complete!")

davinci/gcalendar/migrations/0001_initial.py

@@ -0,0 +1,47 @@
+# Generated by Django 3.1.3 on 2020-11-12 00:09
+
+import datetime
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('caldav', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GoogleAuthInfo',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('email', models.CharField(max_length=191, unique=True)),
+                ('_creds', models.BinaryField()),
+            ],
+            options={
+                'verbose_name': 'Google Authentication Information',
+                'verbose_name_plural': 'Google Authentication Information',
+            },
+        ),
+        migrations.CreateModel(
+            name='GCalSync',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=191, verbose_name='Sync Name')),
+                ('calendar_id', models.CharField(help_text="Can be found in the Google Calendar webapp, under 'Calendar Settings' > 'Integrate calendar' > 'Calendar ID'", max_length=191, verbose_name='Google Calendar ID')),
+                ('purge', models.BooleanField(default=False, help_text='Do not use if you are importing multiple ICS files into the same calendar.', verbose_name='Purge calendar (remove old events)')),
+                ('sync_interval', models.DurationField(default=datetime.timedelta(seconds=21600), verbose_name='Sync interval')),
+                ('last_sync', models.DateTimeField(blank=True, null=True, verbose_name='Last synchronised')),
+                ('active', models.BooleanField(default=True, verbose_name='Sync active')),
+                ('google_account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gcalendar.googleauthinfo', verbose_name='Google Account')),
+                ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='caldav.caldavcalendar', verbose_name='Target calendar')),
+            ],
+            options={
+                'verbose_name': 'Google Calendar Sync',
+                'verbose_name_plural': 'Google Calendar Syncs',
+            },
+        ),
+    ]

davinci/gcalendar/models.py

@@ -0,0 +1,90 @@
+import pickle
+
+import requests
+from django.db import models
+from google.auth.transport.urllib3 import Request
+from requests_oauthlib import OAuth2
+from urllib3 import PoolManager
+
+import logging
+from datetime import timedelta
+
+import humanize
+
+from davinci.caldav.models import CalDAVCalendar
+from davinci.icalendar import ical_utils
+
+
+class GoogleAuthInfo(models.Model):
+    email = models.CharField(max_length=191, unique=True)
+    _creds = models.BinaryField()
+
+    def set_data(self, data):
+        self._creds = pickle.dumps(data)
+
+    def get_data(self):
+        credentials = pickle.loads(self._creds)
+        if credentials and credentials.expired and credentials.refresh_token:
+            credentials.refresh(Request(http=PoolManager()))
+            self._creds = pickle.dumps(credentials)
+            self.save()
+        return pickle.loads(self._creds)
+
+    creds = property(get_data, set_data)
+
+    def __str__(self):
+        return self.email
+
+    class Meta:
+        verbose_name = "Google Authentication Information"
+        verbose_name_plural = "Google Authentication Information"
+
+
+class GCalSync(models.Model):
+    name = models.CharField(max_length=191, verbose_name="Sync Name")
+    calendar_id = models.CharField(max_length=191, verbose_name="Google Calendar ID",
+                                   help_text="Can be found in the Google Calendar webapp, under "
+                                             "'Calendar Settings' > 'Integrate calendar' > 'Calendar ID'")
+    google_account = models.ForeignKey(to=GoogleAuthInfo, on_delete=models.CASCADE, verbose_name="Google Account")
+    target = models.ForeignKey(to=CalDAVCalendar, on_delete=models.CASCADE, verbose_name="Target calendar")
+    purge = models.BooleanField(verbose_name="Purge calendar (remove old events)", default=False,
+                                help_text="Do not use if you are importing multiple ICS files into the same calendar.")
+    sync_interval = models.DurationField(verbose_name="Sync interval", default=timedelta(hours=6))
+    last_sync = models.DateTimeField(verbose_name="Last synchronised", blank=True, null=True)
+    active = models.BooleanField(verbose_name="Sync active", default=True)
+
+    class Meta:
+        verbose_name = "Google Calendar Sync"
+        verbose_name_plural = "Google Calendar Syncs"
+
+    def __str__(self):
+        return f'{self.name} to {self.target} every {humanize.precisedelta(self.sync_interval)} via {self.google_account}'
+
+    @property
+    def humanized_sync_interval(self):
+        return humanize.precisedelta(self.sync_interval)
+
+    def get_ical_data(self):
+        logger = logging.getLogger("davinci.gcalendar.GCalSync.get_ical_data")
+        creds = self.google_account.creds
+        auth = OAuth2(client_id=creds.client_id, token={'access_token': creds.token, 'token_type': 'Bearer'})
+        resp = requests.get(f"https://apidata.googleusercontent.com/caldav/v2/{self.calendar_id}/events", auth=auth)
+        if resp.status_code == 200:
+            return resp.content
+        else:
+            logger.error(f"Error while requesting events from Google for {self.name}: <HTTP {resp.status_code}> {resp.content}")
+
+
+    def get_events(self):
+        logger = logging.getLogger("davinci.gcalendar.GCalSync.get_events")
+        data = self.get_ical_data()
+        try:
+            events = ical_utils.split_events(data)
+        except ValueError as e:
+            logger.error(f"ValueError while parsing events from iCal {self.name}: {e}.")
+            return []
+
+        # We have to pad the uids with some string related to the target calendar,
+        # So that if someone wants to sync the same .ics to two calendars, the UIDs are different.
+        fixed_events = ical_utils.fix_ical_uids(events, self.target)
+        return fixed_events

davinci/gcalendar/templates/admin/gcalendar/googleauthinfo/change_list.html

@@ -0,0 +1,10 @@
+{% extends "admin/change_list.html" %}
+{% block object-tools-items %}
+
+    {{ block.super }}
+
+    <li>
+        <a href="{% url 'auth' %}" class="grp-state-focus addlink">Add Google Account</a>
+    </li>
+
+{% endblock %}

davinci/gcalendar/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

davinci/gcalendar/urls.py

@@ -0,0 +1,23 @@
+"""davinci URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/3.1/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.urls import path
+
+from davinci.gcalendar.views import oauth2_view, oauth2_callback_view
+
+urlpatterns = [
+    path('auth/', oauth2_view, name="auth"),
+    path('oauth2callback/', oauth2_callback_view, name="oauth2callback"),
+]

davinci/gcalendar/views.py

@@ -0,0 +1,39 @@
+from django.conf import settings
+from django.contrib import messages
+from django.shortcuts import reverse, redirect
+from django.contrib.auth import get_user_model
+from google.oauth2 import id_token
+from google_auth_oauthlib.flow import Flow
+from google.auth.transport.urllib3 import Request
+from urllib3 import PoolManager
+
+from davinci.gcalendar.models import GoogleAuthInfo
+
+User = get_user_model()
+
+flow = Flow.from_client_secrets_file(client_secrets_file=settings.GOOGLE_CLIENT_SECRET_FILE,
+                                     scopes=settings.GOOGLE_AUTH_SCOPES)
+
+
+def oauth2_view(request):
+    callback_url = reverse("oauth2callback")
+    flow.redirect_uri = request.build_absolute_uri(callback_url)
+    flow.oauth2session.redirect_uri = flow.redirect_uri
+    auth_url, _ = flow.authorization_url(access_type='offline', prompt='consent', include_granted_scopes='true')
+    return redirect(auth_url)
+
+
+def oauth2_callback_view(request):
+    flow.fetch_token(authorization_response=request.build_absolute_uri().replace('http:', 'https:'))
+    creds = flow.credentials
+    userinfo = id_token.verify_oauth2_token(creds._id_token, Request(http=PoolManager()), creds._client_id)
+    try:
+        gauth = GoogleAuthInfo.objects.get(email=userinfo['email'])
+        gauth.creds = creds
+        gauth.save()
+    except GoogleAuthInfo.DoesNotExist:
+        GoogleAuthInfo.objects.create(email=userinfo['email'], creds=creds)
+
+    # Return Response as you want or Redirect to some URL
+    messages.info(request, f"Google account '{userinfo['email']}' added!")
+    return redirect("admin:index")

davinci/icalendar/forms.py

@@ -4,6 +4,7 @@ from davinci.icalendar.models import ICalSync
 
 from durationwidget.widgets import TimeDurationWidget
 
+
 class ICalSyncForm(forms.ModelForm):
     class Meta:
         model = ICalSync

davinci/icalendar/migrations/0002_auto_20201111_2251.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.1.3 on 2020-11-11 22:51
+
+import datetime
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('icalendar', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='icalsync',
+            name='sync_interval',
+            field=models.DurationField(default=datetime.timedelta(seconds=21600), verbose_name='Sync interval'),
+        ),
+    ]

davinci/icalendar/models.py

@@ -14,7 +14,7 @@ class ICalSync(models.Model):
     target = models.ForeignKey(to=CalDAVCalendar, on_delete=models.CASCADE, verbose_name="Target calendar")
     purge = models.BooleanField(verbose_name="Purge calendar (remove old events)", default=False,
                                 help_text="Do not use if you are importing multiple ICS files into the same calendar.")
-    sync_interval = models.DurationField(verbose_name="Sync interval", default=timedelta(hours=1))
+    sync_interval = models.DurationField(verbose_name="Sync interval", default=timedelta(hours=6))
     last_sync = models.DateTimeField(verbose_name="Last synchronised", blank=True, null=True)
     active = models.BooleanField(verbose_name="Sync active", default=True)
 

davinci/settings.py

@@ -9,6 +9,7 @@ https://docs.djangoproject.com/en/3.1/topics/settings/
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/3.1/ref/settings/
 """
+import os
 import sys
 from pathlib import Path
 
@@ -43,6 +44,7 @@ INSTALLED_APPS = [
 
     'davinci.caldav',
     'davinci.icalendar',
+    'davinci.gcalendar',
 ]
 
 MIDDLEWARE = [
@@ -125,6 +127,15 @@ USE_TZ = True
 # https://docs.djangoproject.com/en/3.1/howto/static-files/
 STATIC_URL = '/static/'
 
+GOOGLE_CLIENT_SECRET_FILE = os.path.join(BASE_DIR, 'credentials.json')
+GOOGLE_AUTH_SCOPES = [
+    'https://www.googleapis.com/auth/userinfo.profile',
+    'openid',
+    'https://www.googleapis.com/auth/calendar.readonly',
+    'https://www.googleapis.com/auth/userinfo.email',
+    'https://www.googleapis.com/auth/calendar.events.readonly',
+]
+
 try:
     from davinci.local import *
 except ImportError:

davinci/urls.py

@@ -14,10 +14,11 @@ Including another URLconf
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 from django.contrib import admin
-from django.urls import path
+from django.urls import path, include
 from django.views.generic import RedirectView
 
 urlpatterns = [
     path('admin/', admin.site.urls),
-    path('', RedirectView.as_view(pattern_name='admin:index'))
+    path('', RedirectView.as_view(pattern_name='admin:index')),
+    path('gcalendar/', include('davinci.gcalendar.urls'))
 ]

requirements.txt

@@ -5,3 +5,4 @@ humanize>=3.1.0,<3.2
 ics>=0.7,<0.8
 caldav>=0.7.1,<0.8
 lockfile>=0.12.2,<0.13
+requests-oauthlib