remember the sync token to avoid reloading the whole history

This commit is contained in:
histausse 2021-10-03 18:02:30 +02:00
parent 0a96845fdf
commit 75d1683b38
Signed by: histausse
GPG key ID: 67486F107F62E9E9
4 changed files with 45 additions and 8 deletions

3
.gitignore vendored
View file

@ -137,3 +137,6 @@ dmypy.json
# Cython debug symbols # Cython debug symbols
cython_debug/ cython_debug/
# Project specific files:
.sync_token

View file

@ -1 +1,2 @@
matrix-nio==0.18.7 matrix-nio==0.18.7
aiopath==0.5.12

View file

@ -11,7 +11,8 @@ packages = matrix_bot
python_requires = >=3.9.2 python_requires = >=3.9.2
package_dir = =src package_dir = =src
install_requires = install_requires =
matrix-nio>=0.18.7 matrix-nio>=0.18.7
aiopath>=0.5.12
[options.extras_require] [options.extras_require]
testing = testing =

View file

@ -5,6 +5,7 @@ Connect to the matrix server and handle interactions with the server.
import asyncio import asyncio
import nio import nio
from aiopath import AsyncPath
from typing import ( from typing import (
Optional, Optional,
NoReturn, NoReturn,
@ -32,6 +33,9 @@ class Client(Aobject):
__client: nio.AsyncClient __client: nio.AsyncClient
__rooms_by_aliases: dict[RoomAlias, Room] __rooms_by_aliases: dict[RoomAlias, Room]
__rooms_by_id: dict[RoomId, Room] __rooms_by_id: dict[RoomId, Room]
__sync_token_file: AsyncPath
__sync_token:Optional[str]
__sync_token_queue:asyncio.Queue[str]
allowed_rooms: Optional[dict[RoomId, Room]] allowed_rooms: Optional[dict[RoomId, Room]]
async def __init__( async def __init__(
@ -39,7 +43,8 @@ class Client(Aobject):
username: str, username: str,
homeserver: str, homeserver: str,
password: str, password: str,
allowed_rooms_names: Optional[list[Union[RoomAlias, RoomId]]]=None allowed_rooms_names: Optional[list[Union[RoomAlias, RoomId]]]=None,
sync_token_file:str="./.sync_token",
): ):
""" """
Initialize the Client. Initialize the Client.
@ -49,6 +54,9 @@ class Client(Aobject):
allowed_rooms: the list of the rooms where the bot is allowed to connect allowed_rooms: the list of the rooms where the bot is allowed to connect
(given by room id (expl: '!xxx:matrix.org') of room alias (expl: (given by room id (expl: '!xxx:matrix.org') of room alias (expl:
'#xxx:matrix.org')) '#xxx:matrix.org'))
sync_token_file: the file where is stored the last sync token received from
the sync responses. This token avoid reloadind all the history of the
bot each time we start it.
""" """
self.__client = nio.AsyncClient( self.__client = nio.AsyncClient(
homeserver, homeserver,
@ -56,6 +64,13 @@ class Client(Aobject):
) )
self.__rooms_by_aliases = {} self.__rooms_by_aliases = {}
self.__rooms_by_id = {} self.__rooms_by_id = {}
self.__sync_token_file = AsyncPath(sync_token_file)
if (await self.__sync_token_file.is_file()):
async with self.__sync_token_file.open() as file:
self.__sync_token = (await file.read()).strip()
else:
self.__sync_token = None
self.__sync_token_queue = asyncio.Queue()
resp = await self.__client.login(password) resp = await self.__client.login(password)
if isinstance(resp, nio.responses.LoginError): if isinstance(resp, nio.responses.LoginError):
raise RuntimeError(f"Fail to connect: {resp.message}") raise RuntimeError(f"Fail to connect: {resp.message}")
@ -109,29 +124,46 @@ class Client(Aobject):
self.__rooms_by_aliases[room_name] = room self.__rooms_by_aliases[room_name] = room
return room return room
async def sync( async def __save_sync_tocken(self)->NoReturn:
"""
Save the sync token.
"""
while True:
token = await self.__sync_token_queue.get()
async with self.__sync_token_file.open(mode='w') as file:
await file.write(token)
self.__sync_token_queue.task_done()
async def __sync(
self, self,
sync_delta:int=30 sync_timeout:int=30
)->NoReturn: )->NoReturn:
""" """
Sync with the server every sync_delta seconds. Sync with the server every sync_delta seconds.
sync_delta: the time in sec between each sync. sync_timeout: the max time in sec between each sync.
""" """
while True: while True:
sync_resp = await self.__client.sync(sync_delta*1000) sync_resp = await self.__client.sync(
sync_delta*1000,
since=self.__sync_token
)
if isinstance(sync_resp, nio.responses.SyncError): if isinstance(sync_resp, nio.responses.SyncError):
print(f"Error while syncronizing: {sync_resp.message}") # TODO: use proper logging print(f"Error while syncronizing: {sync_resp.message}") # TODO: use proper logging
continue continue
self.__sync_token = sync_resp.next_batch
await self.__sync_token_queue.put(self.__sync_token)
async def run( async def run(
self, self,
sync_delta:int=30 sync_timeout:int=30
)->NoReturn: )->NoReturn:
""" """
Run the bot: sync with the server and execute callbacks. Run the bot: sync with the server and execute callbacks.
sync_timeout: the max time in sec between each sync.
""" """
await asyncio.gather( await asyncio.gather(
self.sync(sync_delta=sync_delta) self.__sync(sync_delta=sync_delta),
self.__save_sync_tocken()
) )