aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--README.md6
-rw-r--r--cloud_mdir_sync/config.py5
-rw-r--r--cloud_mdir_sync/credsrv.py65
-rw-r--r--cloud_mdir_sync/gmail.py34
-rw-r--r--cloud_mdir_sync/oauth.py25
-rw-r--r--cloud_mdir_sync/office365.py29
-rw-r--r--doc/example-exim4.conf134
-rw-r--r--doc/smtp.md66
8 files changed, 349 insertions, 15 deletions
diff --git a/README.md b/README.md
index 6c20d0e..e20af76 100644
--- a/README.md
+++ b/README.md
@@ -255,6 +255,12 @@ For mutt use the following configuration:
set maildir_trash = yes
```
+# Mail Delivery Agent Configuration
+
+CMS includes a OAUTH broker than can export a SMTP access token to local SMTP
+delivery agents. The [Outbound mail through SMTP](doc/smtp.md) page describes
+this configuration.
+
# Future Work/TODO
- Use delta queries on mailboxes with MS Graph. Delta queries allow
downloading only changed message meta-data and will accelerate polling of
diff --git a/cloud_mdir_sync/config.py b/cloud_mdir_sync/config.py
index 88a16b6..79ab4a9 100644
--- a/cloud_mdir_sync/config.py
+++ b/cloud_mdir_sync/config.py
@@ -116,5 +116,10 @@ class Config(object):
self.local_mboxes.append(MailDirMailbox(self, directory))
return self.local_mboxes[-1]
+ def CredentialServer(self, path: str, accounts: List, umask=0o600):
+ from .credsrv import CredentialServer
+ self.async_tasks.append(CredentialServer(self, path, accounts, umask))
+ return self.async_tasks[-1]
+
def _direct_message(self, msg):
return self.local_mboxes[0]
diff --git a/cloud_mdir_sync/credsrv.py b/cloud_mdir_sync/credsrv.py
new file mode 100644
index 0000000..97357bc
--- /dev/null
+++ b/cloud_mdir_sync/credsrv.py
@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: GPL-2.0+
+import asyncio
+import contextlib
+import os
+import re
+from typing import List
+
+from . import config, oauth
+
+
+class CredentialServer(object):
+ """Serve XOAUTH2 bearer tokens over a unix domain socket. The client
+ writes the user to obtain a token for and the server responds with the
+ token"""
+ def __init__(self,
+ cfg: config.Config,
+ path: str,
+ accounts: List[oauth.Account],
+ umask=0o600):
+ self.cfg = cfg
+ self.path = os.path.abspath(os.path.expanduser(path))
+ self.umask = umask
+ self.accounts = {}
+ for I in accounts:
+ I.oauth_smtp = True
+ self.accounts[I.user] = I
+
+ async def go(self):
+ old_umask = os.umask(self.umask)
+ try:
+ self.server = await asyncio.start_unix_server(
+ self.handle_client, self.path)
+ finally:
+ os.umask(old_umask)
+ os.chmod(self.path, self.umask)
+
+ async def close(self):
+ pass
+
+ async def handle_client(self, reader: asyncio.StreamReader,
+ writer: asyncio.StreamWriter) -> None:
+ with contextlib.closing(writer):
+ req = await reader.read()
+ g = re.match(r"([^ ,]+)(?:,(\S+))? (\S+@\S+)", req.decode())
+ if g is None:
+ self.cfg.logger.error(f"Invalid credential request {req!r}")
+ return
+ proto, opts_str, user = g.groups()
+ if opts_str:
+ opts = opts_str.split(',')
+ else:
+ opts = []
+
+ self.cfg.logger.debug(
+ f"Credential request {proto!r} {opts} {user!r}")
+
+ account = self.accounts.get(user)
+ if account is None:
+ return
+
+ xoauth2 = await account.get_xoauth2_bytes("SMTP")
+ if xoauth2 is None:
+ return
+ writer.write(xoauth2)
+ await writer.drain()
diff --git a/cloud_mdir_sync/gmail.py b/cloud_mdir_sync/gmail.py
index b22291f..b6d1490 100644
--- a/cloud_mdir_sync/gmail.py
+++ b/cloud_mdir_sync/gmail.py
@@ -14,7 +14,7 @@ import aiohttp
import oauthlib
import requests_oauthlib
-from . import config, mailbox, messages, util
+from . import config, mailbox, messages, oauth, util
from .util import asyncio_complete
@@ -108,7 +108,7 @@ def _retry_protect(func):
return async_wrapper
-class GmailAPI(object):
+class GmailAPI(oauth.Account):
"""An OAUTH2 authenticated session to the Google gmail API"""
# From ziepe.ca
client_id = "14979213351-bik90v3b8b9f22160ura3oah71u3l113.apps.googleusercontent.com"
@@ -119,9 +119,8 @@ class GmailAPI(object):
headers: Optional[Dict[str, str]] = None
def __init__(self, cfg: config.Config, user: str):
+ super().__init__(cfg, user)
self.domain_id = f"gmail-{user}"
- self.cfg = cfg
- self.user = user
async def go(self):
cfg = self.cfg
@@ -130,18 +129,22 @@ class GmailAPI(object):
self.session = aiohttp.ClientSession(connector=connector,
raise_for_status=False)
+ scopes = [
+ "https://www.googleapis.com/auth/gmail.modify",
+ ]
+ if self.oauth_smtp:
+ scopes.append("https://mail.google.com/")
+
self.redirect_url = cfg.web_app.url + "oauth2/gmail"
self.api_token = cfg.msgdb.get_authenticator(self.domain_id)
+ if not oauth.check_scopes(self.api_token, scopes):
+ self.api_token = None
self.oauth = requests_oauthlib.OAuth2Session(
client_id=self.client_id,
client=NativePublicApplicationClient(self.client_id),
redirect_uri=self.redirect_url,
token=self.api_token,
- scope=[
- "https://www.googleapis.com/auth/gmail.modify",
- # This one is needed for SMTP ?
- #"https://mail.google.com/",
- ])
+ scope=scopes)
if self.api_token:
self._set_token()
@@ -296,6 +299,19 @@ class GmailAPI(object):
async def close(self):
await self.session.close()
+ async def get_xoauth2_bytes(self, proto: str) -> bytes:
+ """Return the xoauth2 byte string for the given protocol to login to
+ this account."""
+ while self.api_token is None:
+ await self.authenticate()
+
+ if proto == "SMTP":
+ res = 'user=%s\1auth=%s %s\1\1' % (self.user,
+ self.api_token["token_type"],
+ self.api_token["access_token"])
+ return res.encode()
+ return None
+
class GMailMessage(messages.Message):
gmail_labels: Optional[Set[str]] = None
diff --git a/cloud_mdir_sync/oauth.py b/cloud_mdir_sync/oauth.py
index 163dcba..55f3f31 100644
--- a/cloud_mdir_sync/oauth.py
+++ b/cloud_mdir_sync/oauth.py
@@ -1,10 +1,35 @@
# SPDX-License-Identifier: GPL-2.0+
import asyncio
import os
+from abc import abstractmethod
+from typing import TYPE_CHECKING, List
import aiohttp
import aiohttp.web
+if TYPE_CHECKING:
+ from . import config
+
+
+def check_scopes(token, required_scopes: List[str]) -> bool:
+ if token is None:
+ return False
+ tscopes = set(token.get("scope", []))
+ return set(required_scopes).issubset(tscopes)
+
+
+class Account(object):
+ """An OAUTH2 account"""
+ oauth_smtp = False
+
+ def __init__(self, cfg: "config.Config", user: str):
+ self.cfg = cfg
+ self.user = user
+
+ @abstractmethod
+ async def get_xoauth2_bytes(self, proto: str) -> bytes:
+ pass
+
class WebServer(object):
"""A small web server is used to manage oauth requests. The user should point a browser
diff --git a/cloud_mdir_sync/office365.py b/cloud_mdir_sync/office365.py
index 09d8492..4945724 100644
--- a/cloud_mdir_sync/office365.py
+++ b/cloud_mdir_sync/office365.py
@@ -12,7 +12,7 @@ from typing import Any, Dict, Optional, Union
import aiohttp
import requests
-from . import config, mailbox, messages, util
+from . import config, mailbox, messages, oauth, util
from .util import asyncio_complete
@@ -67,21 +67,20 @@ def _retry_protect(func):
return async_wrapper
-class GraphAPI(object):
+class GraphAPI(oauth.Account):
"""An OAUTH2 authenticated session to the Microsoft Graph API"""
graph_scopes = [
"https://graph.microsoft.com/User.Read",
"https://graph.microsoft.com/Mail.ReadWrite"
]
- graph_token = None
+ graph_token: Optional[Dict[str,str]] = None
owa_scopes = ["https://outlook.office.com/mail.read"]
owa_token = None
authenticator = None
def __init__(self, cfg: config.Config, user: str, tenant: str):
+ super().__init__(cfg, user)
self.domain_id = f"o365-{user}-{tenant}"
- self.cfg = cfg
- self.user = user
self.tenant = tenant
if self.user is not None:
@@ -93,7 +92,7 @@ class GraphAPI(object):
# with our caching scheme. See
# https://docs.microsoft.com/en-us/graph/outlook-immutable-id
self.headers= {"Prefer": 'IdType="ImmutableId"'}
- self.owa_headers = {}
+ self.owa_headers: Dict[str, str] = {}
async def go(self):
import msal
@@ -111,6 +110,11 @@ class GraphAPI(object):
authority=f"https://login.microsoftonline.com/{self.tenant}",
token_cache=self.msl_cache)
+ if self.oauth_smtp:
+ self.owa_scopes = self.owa_scopes + [
+ "https://outlook.office.com/SMTP.Send"
+ ]
+
def _cached_authenticate(self):
accounts = self.msal.get_accounts(self.user)
if len(accounts) != 1:
@@ -352,6 +356,19 @@ class GraphAPI(object):
async def close(self):
await self.session.close()
+ async def get_xoauth2_bytes(self, proto: str) -> bytes:
+ """Return the xoauth2 byte string for the given protocol to login to
+ this account."""
+ while self.owa_token is None:
+ await self.authenticate()
+
+ if proto == "SMTP":
+ res = 'user=%s\1auth=%s %s\1\1' % (self.user,
+ self.owa_token["token_type"],
+ self.owa_token["access_token"])
+ return res.encode()
+ return None
+
class O365Mailbox(mailbox.Mailbox):
"""Cloud Office365 mailbox using the Microsoft Graph RESET API for data access"""
diff --git a/doc/example-exim4.conf b/doc/example-exim4.conf
new file mode 100644
index 0000000..2f2c906
--- /dev/null
+++ b/doc/example-exim4.conf
@@ -0,0 +1,134 @@
+# Specify local domains HERE, these are names that might appear in email
+domainlist local_domains =
+domainlist relay_to_domains =
+hostlist relay_from_hosts = localhost
+# Create this file HERE
+# It has single lines in the format:
+# domain.com: host=smtp.office365.com::587 helo=cms user=user@domain.com oauth=/home/XX/mail/.cms/exim/unix
+# oauth= may also be replaced with password= to do basic authentication. The
+# file is searched based on the Envelope From when invoking sendmail
+SMARTFN = /etc/exim4/exim-smart-hosts
+
+# We don't have IPv6, do not even try.
+disable_ipv6=true
+
+# It is also a good idea to edit /etc/default/exim4 and switch to 'queueonly' mode (Debian)
+local_interfaces = <; [127.0.0.1]:25
+
+acl_smtp_rcpt = acl_check_rcpt
+acl_smtp_data = acl_check_data
+
+tls_advertise_hosts =
+
+# Trusted users are allowed to override the sender envelope
+trusted_users = # Add your user name HERE
+
+never_users = root
+
+host_lookup = *
+
+prdr_enable = true
+
+log_selector = +smtp_protocol_error +smtp_syntax_error \
+ +tls_certificate_verified
+
+ignore_bounce_errors_after = 2d
+timeout_frozen_after = 7d
+
+keep_environment =
+add_environment = <; PATH=/bin:/usr/bin
+
+begin acl
+acl_check_rcpt:
+
+ # Standard input
+ accept hosts = :
+ control = dkim_disable_verify
+
+ deny message = Restricted characters in address
+ domains = +local_domains
+ local_parts = ^[.] : ^.*[@%!/|]
+
+
+ deny message = Restricted characters in address
+ domains = !+local_domains
+ local_parts = ^[./|] : ^.*[@%!] : ^.*/\\.\\./
+
+
+ accept local_parts = postmaster
+ domains = +local_domains
+
+
+ require verify = sender
+
+ accept hosts = +relay_from_hosts
+ control = submission
+ control = dkim_disable_verify
+
+ accept authenticated = *
+ control = submission
+ control = dkim_disable_verify
+
+ require message = relay not permitted
+ domains = +local_domains : +relay_to_domains
+
+ require verify = recipient
+
+ accept
+
+begin routers
+
+# The router is sensitive to the sender address and will use the correct outgoing server.
+# Use something like:
+# exim -f 'user@domain.com' -bt user@otherdomain.com
+# To quick test
+smarthost:
+ debug_print = "R: smarthost for $local_part@$domain"
+ driver = manualroute
+ domains = ! +local_domains
+ transport = remote_smtp_smarthost
+ route_data = ${extract{host}{${lookup{$sender_address_domain}lsearch{SMARTFN}}}}
+ no_more
+
+begin transports
+
+remote_smtp_smarthost:
+ debug_print = "T: remote_smtp_smarthost for $local_part@$domain"
+ driver = smtp
+ helo_data = ${extract{helo}{${lookup{$sender_address_domain}lsearch{SMARTFN}}}{$value}{wakko.ziepe.ca}}
+ hosts_require_auth = ${extract{user}{${lookup{$sender_address_domain}lsearch{SMARTFN}}}{*}{}}
+ hosts_require_tls = *
+ tls_tempfail_tryclear = false
+ tls_verify_certificates = system
+
+begin retry
+
+* * F,2h,15m; G,16h,1h,1.5; F,4d,6h
+
+begin rewrite
+
+# Replace user and domain HERE
+root@+local_domains user@domain.com Ffrs
+user@+local_domains user@domain.com Ffrs
+
+begin authenticators
+
+xoauth2_smart:
+ driver = plaintext
+ client_condition = ${if and {{!eq{$tls_out_cipher}{}} {eq{${extract{oauth}{${lookup{$sender_address_domain}lsearch{SMARTFN}}}{}fail}}{}}} }
+ public_name = XOAUTH2
+ client_ignore_invalid_base64 = true
+ client_send = : ${readsocket{${extract{oauth}{${lookup{$sender_address_domain}lsearch{SMARTFN}}}{$value}fail}}{SMTP ${extract{user}{${lookup{$sender_address_domain}lsearch{SMARTFN}}}{$value}fail}}}
+
+# Plain has fewer round trips, so prefer to use it
+plain_smart:
+ driver = plaintext
+ client_condition = ${if and {{!eq{$tls_out_cipher}{}} {eq{${extract{password}{${lookup{$sender_address_domain}lsearch{SMARTFN}}}{}fail}}{}}} }
+ public_name = PLAIN
+ client_send = ^${extract{user}{${lookup{$sender_address_domain}lsearch{SMARTFN}}}{$value}fail}^${extract{password}{${lookup{$sender_address_domain}lsearch{SMARTFN}}}}
+
+login_smart:
+ driver = plaintext
+ client_condition = ${if and {{!eq{$tls_out_cipher}{}} {eq{${extract{password}{${lookup{$sender_address_domain}lsearch{SMARTFN}}}{}fail}}{}}} }
+ public_name = LOGIN
+ client_send = : ${extract{user}{${lookup{$sender_address_domain}lsearch{SMARTFN}}}{$value}fail} : ${extract{password}{${lookup{$sender_address_domain}lsearch{SMARTFN}}}}
diff --git a/doc/smtp.md b/doc/smtp.md
new file mode 100644
index 0000000..48f3d6c
--- /dev/null
+++ b/doc/smtp.md
@@ -0,0 +1,66 @@
+# Outbound mail through SMTP
+
+The cloud services now all support OAUTH2 as an authentication method for
+SMTP, and CMS provides an internal broker service to acquire and expose the
+OAUTH access token needed for SMTP.
+
+This allows the use of several normal SMTP tools without having to revert
+to BASIC authentication.
+
+## CMS Configuration
+
+CMS uses a UNIX domain socket to expose the access token. CMS must be running
+to maintain a fresh token.
+
+This feature is enabled in the configuration file:
+
+```Python
+account = Office365_Account(user="user@domain.com")
+Office365("inbox", account)
+CredentialServer("/var/run/user/XXX/cms.sock",
+ accounts=[account])
+```
+
+Upon restart CMS will acquire and maintain a OAUTH token with the SMTP scope
+for the specified accounts, and serve token requests on the specified path.
+
+# exim 4
+
+Exim is a long standing UNIX mail system that is fully featured. exim's flexible
+authentication can support the use of OAUTH tokens:
+
+```
+begin authenticators
+
+xoauth2_smart:
+ driver = plaintext
+ client_condition = ${if !eq{$tls_out_cipher}{}}
+ public_name = XOAUTH2
+ client_ignore_invalid_base64 = true
+ client_send = : ${readsocket{/home/XX/mail/.cms/exim/cms.sock}{SMTP user@domain}}
+```
+
+Since exim runs as a system daemon, permissions must be set to allow access to
+the socket:
+
+```sh
+cd /home/XX/mail/.cms
+mkdir exim
+chmod 0750 exim
+sudo chgrp Debian-exim cms
+```
+
+And the CMS configuration must specify a umask:
+
+```Python
+CredentialServer("/home/XX/mail/.cms/exim/cms.sock",
+ accounts=[account],
+ umask=0o666)
+```
+
+A fully functional [exim4.conf](example-exim4.conf) is provided. This minimal,
+relay only config can replace the entire configuration from the distro, after
+making the adjustments noted. In this mode /usr/bin/sendmail will be fully
+functional for outbound mail and if multiple accounts are required, it will
+automatically choose the account to send mail through based on the Envelope
+From header.
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-19 11:06:46 +0000