From 2cab5b300291a2a8e19b92e0d3a63a88ac82628e Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 18 Feb 2021 14:50:01 +0000 Subject: [PATCH 01/58] simplify wait_for --- ircrobots/asyncs.py | 15 +----- ircrobots/server.py | 125 ++++++++++++-------------------------------- 2 files changed, 36 insertions(+), 104 deletions(-) diff --git a/ircrobots/asyncs.py b/ircrobots/asyncs.py index 54d0b3b..7b2f4e2 100644 --- a/ircrobots/asyncs.py +++ b/ircrobots/asyncs.py @@ -19,17 +19,9 @@ class MaybeAwait(Generic[TEvent]): class WaitFor(object): def __init__(self, response: IMatchResponse, - deadline: float): + label: Optional[str]=None): self.response = response - self.deadline = deadline - self._label: Optional[str] = None - self._our_fut: "Future[Line]" = Future() - - def __await__(self) -> Generator[Any, None, Line]: - return self._our_fut.__await__() - - def with_label(self, label: str): - self._label = label + self._label = label def match(self, server: IServer, line: Line): if (self._label is not None and @@ -39,6 +31,3 @@ class WaitFor(object): label == self._label): return True return self.response.match(server, line) - - def resolve(self, line: Line): - self._our_fut.set_result(line) diff --git a/ircrobots/server.py b/ircrobots/server.py index 46da592..a70f355 100644 --- a/ircrobots/server.py +++ b/ircrobots/server.py @@ -28,7 +28,7 @@ from .interface import ITCPTransport, ITCPReader, ITCPWriter THROTTLE_RATE = 4 # lines THROTTLE_TIME = 2 # seconds -PING_TIMEOUT = 60 # seconds +PING_INTERVAL = 60 # seconds WAIT_TIMEOUT = 20 # seconds JOIN_ERR_FIRST = [ @@ -64,7 +64,9 @@ class Server(IServer): self._send_queue: PriorityQueue[SentLine] = PriorityQueue() self.desired_caps: Set[ICapability] = set([]) - self._read_queue: Deque[Line] = deque() + self.read_lock = asyncio.Lock() + self._read_queue: Deque[Line] = deque() + self._process_queue: Deque[Line] = deque() self._wait_for: Optional[Tuple[Awaitable, WaitFor]] = None self._wait_for_fut: Optional[Future[WaitFor]] = None @@ -260,94 +262,44 @@ class Server(IServer): else: await self.send(build("WHO", [chan])) - async def _read_line(self, timeout: float) -> Optional[Line]: + async def _read_line(self) -> Line: while True: - if self._read_queue: - return self._read_queue.popleft() + async with self.read_lock: + if self._read_queue: + return self._read_queue.popleft() - try: - async with timeout_(timeout): - data = await self._reader.read(1024) - except asyncio.TimeoutError: - return None - - self.last_read = monotonic() - lines = self.recv(data) - for line in lines: - self._read_queue.append(line) - - async def _line_or_wait(self, - line_aw: asyncio.Task - ) -> Optional[Tuple[Awaitable, WaitFor]]: - wait_for_fut: Future[WaitFor] = Future() - self._wait_for_fut = wait_for_fut - - done, pend = await asyncio.wait([line_aw, wait_for_fut], - return_when=asyncio.FIRST_COMPLETED) - self._wait_for_fut = None - - if wait_for_fut.done(): - new_line_aw = list(pend)[0] - return (new_line_aw, wait_for_fut.result()) - else: - return None + data = await self._reader.read(1024) + lines = self.recv(data) + # last_read under self.recv() as recv might throw Disconnected + self.last_read = monotonic() + for line in lines: + self._read_queue.append(line) async def _read_lines(self): - waiting_lines: List[Tuple[Line, Optional[Emit]]] = [] sent_ping = False while True: - now = monotonic() - timeouts: List[float] = [] - timeouts.append((self.last_read+PING_TIMEOUT)-now) - if self._wait_for is not None: - _, wait_for = self._wait_for - timeouts.append(wait_for.deadline-now) - - line = await self._read_line(max([0.1, min(timeouts)])) - if line is None: - now = monotonic() - since = now-self.last_read - - if self._wait_for is not None: - aw, wait_for = self._wait_for - if wait_for.deadline <= now: - self._wait_for = None - await aw - - if since >= PING_TIMEOUT: - if since >= (PING_TIMEOUT*2): - raise ServerDisconnectedException() - elif not sent_ping: + if not self._process_queue: + try: + async with timeout_(PING_INTERVAL): + line = await self._read_line() + except asyncio.TimeoutError: + if not sent_ping: sent_ping = True await self.send(build("PING", ["hello"])) continue - else: - sent_ping = False - emit = self.parse_tokens(line) - - waiting_lines.append((line, emit)) - self.line_preread(line) - - if self._wait_for is not None: - aw, wait_for = self._wait_for - if wait_for.match(self, line): - wait_for.resolve(line) - self._wait_for = await self._line_or_wait(aw) - if self._wait_for is not None: - continue else: - continue + raise ServerDisconnectedException() + else: + sent_ping = False + self._process_queue.append(line) - for i in range(len(waiting_lines)): - line, emit = waiting_lines.pop(0) - line_aw = self._on_read(line, emit) - self._wait_for = await self._line_or_wait(line_aw) - if self._wait_for is not None: - break + line = self._process_queue.popleft() + emit = self.parse_tokens(line) + await self._on_read(line, emit) async def wait_for(self, response: Union[IMatchResponse, Set[IMatchResponse]], - sent_aw: Optional[Awaitable[SentLine]]=None, + label: Optional[str]=None, timeout: float=WAIT_TIMEOUT ) -> Line: @@ -357,22 +309,13 @@ class Server(IServer): else: response_obj = response - deadline = monotonic()+timeout - our_wait_for = WaitFor(response_obj, deadline) - if self._wait_for_fut is not None: - self._wait_for_fut.set_result(our_wait_for) - else: - cur_task = asyncio.current_task() - if cur_task is not None: - self._wait_for = (cur_task, our_wait_for) - - if sent_aw is not None: - sent_line = await sent_aw - label = str(sent_line.id) - our_wait_for.with_label(label) - + wait_for = WaitFor(response_obj, label) async with timeout_(timeout): - return (await our_wait_for) + while True: + line = await self._read_line() + self._process_queue.append(line) + if wait_for.match(self, line): + return line async def _on_send_line(self, line: Line): if (line.command in ["PRIVMSG", "NOTICE", "TAGMSG"] and From fc0e8470cc49446c35095517f239a346c419b509 Mon Sep 17 00:00:00 2001 From: jesopo Date: Fri, 26 Mar 2021 12:35:02 +0000 Subject: [PATCH 02/58] change pre-001 throttle to 100 lines in 1 second --- ircrobots/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ircrobots/server.py b/ircrobots/server.py index 46da592..73a61b6 100644 --- a/ircrobots/server.py +++ b/ircrobots/server.py @@ -54,8 +54,7 @@ class Server(IServer): self.disconnected = False - self.throttle = Throttler( - rate_limit=100, period=THROTTLE_TIME) + self.throttle = Throttler(rate_limit=100, period=1) self.sasl_state = SASLResult.NONE self.last_read = monotonic() From d0c6b4a43d2ef4e5221d732dfa008fe8bccc10b7 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 10 Apr 2021 13:54:00 +0000 Subject: [PATCH 03/58] update ircstates to 0.11.8 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e9918bb..170f4cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ anyio ~=2.0.2 asyncio-throttle ~=1.0.1 dataclasses ~=0.6; python_version<"3.7" -ircstates ~=0.11.7 +ircstates ~=0.11.8 async_stagger ~=0.3.0 async_timeout ~=3.0.1 From 90fb4b7bbae27622fa15f81a0f465de8fa4bed10 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 10 Apr 2021 13:55:04 +0000 Subject: [PATCH 04/58] v0.3.8 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0f82685..6678432 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.7 +0.3.8 From 6a05370a12bebc3932eba99ecb349994eab37a34 Mon Sep 17 00:00:00 2001 From: jesopo Date: Wed, 12 May 2021 10:52:39 +0000 Subject: [PATCH 05/58] simplify wait_for --- ircrobots/server.py | 124 ++++++++++++++++---------------------------- requirements.txt | 1 + 2 files changed, 45 insertions(+), 80 deletions(-) diff --git a/ircrobots/server.py b/ircrobots/server.py index 73a61b6..14c8a40 100644 --- a/ircrobots/server.py +++ b/ircrobots/server.py @@ -6,6 +6,7 @@ from collections import deque from time import monotonic import anyio +from asyncio_rlock import RLock from asyncio_throttle import Throttler from async_timeout import timeout as timeout_ from ircstates import Emit, Channel, ChannelUser @@ -63,10 +64,13 @@ class Server(IServer): self._send_queue: PriorityQueue[SentLine] = PriorityQueue() self.desired_caps: Set[ICapability] = set([]) - self._read_queue: Deque[Line] = deque() + self._read_queue: Deque[Line] = deque() + self._process_queue: Deque[Line] = deque() - self._wait_for: Optional[Tuple[Awaitable, WaitFor]] = None - self._wait_for_fut: Optional[Future[WaitFor]] = None + self._read_lguard = RLock() + self.read_lock = self._read_lguard + self._read_lwork = asyncio.Lock() + self._wait_for = asyncio.Event() self._pending_who: Deque[str] = deque() self._alt_nicks: List[str] = [] @@ -273,76 +277,42 @@ class Server(IServer): self.last_read = monotonic() lines = self.recv(data) for line in lines: + self.line_preread(line) self._read_queue.append(line) - async def _line_or_wait(self, - line_aw: asyncio.Task - ) -> Optional[Tuple[Awaitable, WaitFor]]: - wait_for_fut: Future[WaitFor] = Future() - self._wait_for_fut = wait_for_fut - - done, pend = await asyncio.wait([line_aw, wait_for_fut], - return_when=asyncio.FIRST_COMPLETED) - self._wait_for_fut = None - - if wait_for_fut.done(): - new_line_aw = list(pend)[0] - return (new_line_aw, wait_for_fut.result()) - else: - return None - async def _read_lines(self): - waiting_lines: List[Tuple[Line, Optional[Emit]]] = [] - sent_ping = False + ping_sent = False while True: - now = monotonic() - timeouts: List[float] = [] - timeouts.append((self.last_read+PING_TIMEOUT)-now) - if self._wait_for is not None: - _, wait_for = self._wait_for - timeouts.append(wait_for.deadline-now) + async with self._read_lguard: + pass - line = await self._read_line(max([0.1, min(timeouts)])) - if line is None: - now = monotonic() - since = now-self.last_read + if not self._process_queue: + async with self._read_lwork: + read_aw = self._read_line(PING_TIMEOUT) + dones, notdones = await asyncio.wait( + [read_aw, self._wait_for.wait()], + return_when=asyncio.FIRST_COMPLETED + ) + self._wait_for.clear() - if self._wait_for is not None: - aw, wait_for = self._wait_for - if wait_for.deadline <= now: - self._wait_for = None - await aw + for done in dones: + if isinstance(done.result(), Line): + line = done.result() + self._process_queue.append(line) + elif done.result() is None: + if ping_sent: + await self.send(build("PING", ["hello"])) + ping_sent = True + else: + await self.disconnect() + raise ServerDisconnectedException() + for notdone in notdones: + notdone.cancel() - if since >= PING_TIMEOUT: - if since >= (PING_TIMEOUT*2): - raise ServerDisconnectedException() - elif not sent_ping: - sent_ping = True - await self.send(build("PING", ["hello"])) - continue else: - sent_ping = False + line = self._process_queue.popleft() emit = self.parse_tokens(line) - - waiting_lines.append((line, emit)) - self.line_preread(line) - - if self._wait_for is not None: - aw, wait_for = self._wait_for - if wait_for.match(self, line): - wait_for.resolve(line) - self._wait_for = await self._line_or_wait(aw) - if self._wait_for is not None: - continue - else: - continue - - for i in range(len(waiting_lines)): - line, emit = waiting_lines.pop(0) - line_aw = self._on_read(line, emit) - self._wait_for = await self._line_or_wait(line_aw) - if self._wait_for is not None: - break + await self._on_read(line, emit) async def wait_for(self, response: Union[IMatchResponse, Set[IMatchResponse]], @@ -356,22 +326,16 @@ class Server(IServer): else: response_obj = response - deadline = monotonic()+timeout - our_wait_for = WaitFor(response_obj, deadline) - if self._wait_for_fut is not None: - self._wait_for_fut.set_result(our_wait_for) - else: - cur_task = asyncio.current_task() - if cur_task is not None: - self._wait_for = (cur_task, our_wait_for) - - if sent_aw is not None: - sent_line = await sent_aw - label = str(sent_line.id) - our_wait_for.with_label(label) - - async with timeout_(timeout): - return (await our_wait_for) + async with self._read_lguard: + self._wait_for.set() + async with self._read_lwork: + async with timeout_(timeout): + while True: + line = await self._read_line(timeout) + if line: + self._process_queue.append(line) + if response_obj.match(self, line): + return line async def _on_send_line(self, line: Line): if (line.command in ["PRIVMSG", "NOTICE", "TAGMSG"] and diff --git a/requirements.txt b/requirements.txt index 170f4cc..771bd74 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ anyio ~=2.0.2 +asyncio-rlock ~=0.1.0 asyncio-throttle ~=1.0.1 dataclasses ~=0.6; python_version<"3.7" ircstates ~=0.11.8 From bfb5b4ec61fbd3376fbabe631f5b2eef347ce49c Mon Sep 17 00:00:00 2001 From: jesopo Date: Wed, 12 May 2021 10:54:47 +0000 Subject: [PATCH 06/58] v0.3.9 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6678432..940ac09 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.8 +0.3.9 From 0253aba99e4513334c8d3fd9d3733aadb16d1674 Mon Sep 17 00:00:00 2001 From: jesopo Date: Wed, 12 May 2021 10:58:51 +0000 Subject: [PATCH 07/58] v0.3.10 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 940ac09..5503126 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.9 +0.3.10 From 3574868458cf5415d076981cd5c5b4814961a333 Mon Sep 17 00:00:00 2001 From: jesopo Date: Wed, 12 May 2021 11:24:54 +0000 Subject: [PATCH 08/58] reset ping timer when we read a line --- ircrobots/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ircrobots/server.py b/ircrobots/server.py index 14c8a40..dfc7b62 100644 --- a/ircrobots/server.py +++ b/ircrobots/server.py @@ -297,7 +297,8 @@ class Server(IServer): for done in dones: if isinstance(done.result(), Line): - line = done.result() + ping_sent = False + line = done.result() self._process_queue.append(line) elif done.result() is None: if ping_sent: From a14c7c34a29ae181900a3609af23af6772271b0d Mon Sep 17 00:00:00 2001 From: jesopo Date: Wed, 12 May 2021 11:28:27 +0000 Subject: [PATCH 09/58] v0.3.11 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5503126..2080591 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.10 +0.3.11 From bdfb91b51da65d8d8dad20280ec0e08d00097b6c Mon Sep 17 00:00:00 2001 From: jesopo Date: Wed, 12 May 2021 11:52:33 +0000 Subject: [PATCH 10/58] invert ping check --- ircrobots/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ircrobots/server.py b/ircrobots/server.py index dfc7b62..8a3cc49 100644 --- a/ircrobots/server.py +++ b/ircrobots/server.py @@ -301,7 +301,7 @@ class Server(IServer): line = done.result() self._process_queue.append(line) elif done.result() is None: - if ping_sent: + if not ping_sent: await self.send(build("PING", ["hello"])) ping_sent = True else: From b4eaf6c24cbeb3e0f219ca27b40bc0d4d6ae9c37 Mon Sep 17 00:00:00 2001 From: jesopo Date: Wed, 12 May 2021 11:56:24 +0000 Subject: [PATCH 11/58] v0.3.12 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2080591..0b9c019 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.11 +0.3.12 From 6fddfb7fe9fa445570435e9c8bd26f152ed229a7 Mon Sep 17 00:00:00 2001 From: jesopo Date: Wed, 12 May 2021 12:34:06 +0000 Subject: [PATCH 12/58] reset ping_sent in wait_for too --- ircrobots/server.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ircrobots/server.py b/ircrobots/server.py index 8a3cc49..c14865f 100644 --- a/ircrobots/server.py +++ b/ircrobots/server.py @@ -67,6 +67,7 @@ class Server(IServer): self._read_queue: Deque[Line] = deque() self._process_queue: Deque[Line] = deque() + self._ping_sent = False self._read_lguard = RLock() self.read_lock = self._read_lguard self._read_lwork = asyncio.Lock() @@ -281,7 +282,6 @@ class Server(IServer): self._read_queue.append(line) async def _read_lines(self): - ping_sent = False while True: async with self._read_lguard: pass @@ -297,13 +297,12 @@ class Server(IServer): for done in dones: if isinstance(done.result(), Line): - ping_sent = False - line = done.result() - self._process_queue.append(line) + self._ping_sent = False + self._process_queue.append(done.result()) elif done.result() is None: - if not ping_sent: + if not self._ping_sent: await self.send(build("PING", ["hello"])) - ping_sent = True + self._ping_sent = True else: await self.disconnect() raise ServerDisconnectedException() @@ -334,6 +333,7 @@ class Server(IServer): while True: line = await self._read_line(timeout) if line: + self._ping_sent = False self._process_queue.append(line) if response_obj.match(self, line): return line From f22471993a01f5a06b933488005663001cd8002a Mon Sep 17 00:00:00 2001 From: jesopo Date: Wed, 12 May 2021 12:35:35 +0000 Subject: [PATCH 13/58] v0.3.13 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0b9c019..e473765 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.12 +0.3.13 From dd41b0dbde6849ec71d60a90dfd8682cfa83c885 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 22 May 2021 08:43:11 +0000 Subject: [PATCH 14/58] parse tokens in wait_for - waity things expect state change --- ircrobots/server.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ircrobots/server.py b/ircrobots/server.py index c14865f..5a383a1 100644 --- a/ircrobots/server.py +++ b/ircrobots/server.py @@ -65,7 +65,7 @@ class Server(IServer): self.desired_caps: Set[ICapability] = set([]) self._read_queue: Deque[Line] = deque() - self._process_queue: Deque[Line] = deque() + self._process_queue: Deque[Tuple[Line, Optional[Emit]]] = deque() self._ping_sent = False self._read_lguard = RLock() @@ -298,7 +298,9 @@ class Server(IServer): for done in dones: if isinstance(done.result(), Line): self._ping_sent = False - self._process_queue.append(done.result()) + line = done.result() + emit = self.parse_tokens(line) + self._process_queue.append((line, emit)) elif done.result() is None: if not self._ping_sent: await self.send(build("PING", ["hello"])) @@ -310,8 +312,7 @@ class Server(IServer): notdone.cancel() else: - line = self._process_queue.popleft() - emit = self.parse_tokens(line) + line, emit = self._process_queue.popleft() await self._on_read(line, emit) async def wait_for(self, @@ -334,7 +335,8 @@ class Server(IServer): line = await self._read_line(timeout) if line: self._ping_sent = False - self._process_queue.append(line) + emit = self.parse_tokens(line) + self._process_queue.append((line, emit)) if response_obj.match(self, line): return line From 930342d74fbd7c655f8d0cca00cbe7b70b60904b Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 22 May 2021 08:43:50 +0000 Subject: [PATCH 15/58] v0.3.14 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e473765..0b69c00 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.13 +0.3.14 From 8d3681eba1ff684dd94a38a5fd7f20b90f16b0f3 Mon Sep 17 00:00:00 2001 From: jesopo Date: Mon, 24 May 2021 18:08:26 +0000 Subject: [PATCH 16/58] freenode is dead long live libera.chat --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c8e88cd..409a2f5 100644 --- a/README.md +++ b/README.md @@ -11,4 +11,4 @@ see [examples/](examples/) for some usage demonstration. ## contact -Come say hi at [##irctokens on freenode](https://webchat.freenode.net/?channels=%23%23irctokens) +Come say hi at `#irctokens` on irc.libera.chat From ab17645d832e8a16971525d75e570eabc7d98390 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 26 Jun 2021 15:08:48 +0000 Subject: [PATCH 17/58] catch reconnection failures, do exponential backoff --- ircrobots/bot.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/ircrobots/bot.py b/ircrobots/bot.py index cdfe3ab..3a29f7f 100644 --- a/ircrobots/bot.py +++ b/ircrobots/bot.py @@ -1,4 +1,4 @@ -import asyncio +import asyncio, traceback import anyio from typing import Dict @@ -13,20 +13,30 @@ class Bot(IBot): self.servers: Dict[str, Server] = {} self._server_queue: asyncio.Queue[Server] = asyncio.Queue() - # methods designed to be overridden def create_server(self, name: str): return Server(self, name) + async def disconnected(self, server: IServer): if (server.name in self.servers and server.params is not None and server.disconnected): - await asyncio.sleep(server.params.reconnect) - await self.add_server(server.name, server.params) - # /methods designed to be overridden + + reconnect = server.params.reconnect + + while True: + await asyncio.sleep(reconnect) + try: + await self.add_server(server.name, server.params) + except Exception as e: + traceback.print_exc() + # let's try again, exponential backoff up to 5 mins + reconnect = min(reconnect*2, 300) + else: + break async def disconnect(self, server: IServer): - await server.disconnect() del self.servers[server.name] + await server.disconnect() async def add_server(self, name: str, params: ConnectionParams) -> Server: server = self.create_server(name) From fb93d59c434ff424eaf594354159b8b417b09022 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 26 Jun 2021 15:11:32 +0000 Subject: [PATCH 18/58] v0.4.0 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0b69c00..1d0ba9e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.14 +0.4.0 From 64935c7a8d406faf47f84c7abe843e612395e3a6 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 11 Sep 2021 15:40:18 +0000 Subject: [PATCH 19/58] react to pre-reg ERR_ERRONEUSNICKNAME the same as ERR_NICKNAMEINUSE --- ircrobots/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ircrobots/server.py b/ircrobots/server.py index 5a383a1..e8ff4a7 100644 --- a/ircrobots/server.py +++ b/ircrobots/server.py @@ -182,7 +182,7 @@ class Server(IServer): self._pending_who.popleft() await self._next_who() - elif (line.command == ERR_NICKNAMEINUSE and + elif (line.command in {ERR_NICKNAMEINUSE, ERR_ERRONEUSNICKNAME} and not self.registered): if self._alt_nicks: nick = self._alt_nicks.pop(0) From c7604686a23fe50c4a186955f4cf3a16c8a33aa2 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 11 Sep 2021 15:42:47 +0000 Subject: [PATCH 20/58] channel_user.modes is now a set --- ircrobots/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ircrobots/server.py b/ircrobots/server.py index e8ff4a7..d916761 100644 --- a/ircrobots/server.py +++ b/ircrobots/server.py @@ -554,7 +554,7 @@ class Server(IServer): for symbol in symbols: mode = self.isupport.prefix.from_prefix(symbol) if mode is not None: - channel_user.modes.append(mode) + channel_user.modes.add(mode) obj.channels.append(channel_user) elif line.command == RPL_ENDOFWHOIS: From 8ee692f1beb188505d23d2d0abb9e884ae58368f Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 11 Sep 2021 15:43:40 +0000 Subject: [PATCH 21/58] upgrade ircstates to 0.11.9 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 771bd74..358c086 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,6 @@ anyio ~=2.0.2 asyncio-rlock ~=0.1.0 asyncio-throttle ~=1.0.1 dataclasses ~=0.6; python_version<"3.7" -ircstates ~=0.11.8 +ircstates ~=0.11.9 async_stagger ~=0.3.0 async_timeout ~=3.0.1 From bb87c86b378a603834655a822cef9576bc796f64 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 11 Sep 2021 15:44:15 +0000 Subject: [PATCH 22/58] v0.4.1 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1d0ba9e..267577d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.0 +0.4.1 From a03f11449c3212e2d585e23abd035950dd1f294b Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 18 Sep 2021 17:11:40 +0000 Subject: [PATCH 23/58] upgrade ircstates to v0.11.10 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 358c086..9437d30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,6 @@ anyio ~=2.0.2 asyncio-rlock ~=0.1.0 asyncio-throttle ~=1.0.1 dataclasses ~=0.6; python_version<"3.7" -ircstates ~=0.11.9 +ircstates ~=0.11.10 async_stagger ~=0.3.0 async_timeout ~=3.0.1 From 9ca1ec21c939ff4b32567eef50f3aa639aad10dd Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 18 Sep 2021 17:15:53 +0000 Subject: [PATCH 24/58] v0.4.2 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 267577d..2b7c5ae 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.1 +0.4.2 From ab65e39ab90c77c8fbfbf11782ca430de12c49f0 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 18 Sep 2021 17:34:52 +0000 Subject: [PATCH 25/58] handle ERR_SASLABORTED --- ircrobots/sasl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ircrobots/sasl.py b/ircrobots/sasl.py index 887484c..584f61c 100644 --- a/ircrobots/sasl.py +++ b/ircrobots/sasl.py @@ -32,7 +32,9 @@ AUTH_BYTE_MAX = 400 AUTHENTICATE_ANY = Response("AUTHENTICATE", [ANY]) NUMERICS_FAIL = Response(ERR_SASLFAIL) -NUMERICS_INITIAL = Responses([ERR_SASLFAIL, ERR_SASLALREADY, RPL_SASLMECHS]) +NUMERICS_INITIAL = Responses([ + ERR_SASLFAIL, ERR_SASLALREADY, RPL_SASLMECHS, ERR_SASLABORTED +]) NUMERICS_LAST = Responses([RPL_SASLSUCCESS, ERR_SASLFAIL]) def _b64e(s: str): From dfd78b3d3eda8b1a3f9790b8337133fc606d4026 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sun, 19 Sep 2021 21:32:02 +0000 Subject: [PATCH 26/58] v0.4.3 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2b7c5ae..17b2ccd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.2 +0.4.3 From 7b6a84592742f731467609d836ae71b76ace4366 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sun, 19 Sep 2021 21:34:57 +0000 Subject: [PATCH 27/58] don't infinitely loop SASLUserPass attempts on FAIL or ABORT --- ircrobots/sasl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ircrobots/sasl.py b/ircrobots/sasl.py index 584f61c..8f3e21c 100644 --- a/ircrobots/sasl.py +++ b/ircrobots/sasl.py @@ -152,6 +152,8 @@ class SASLContext(ServerContext): return SASLResult.SUCCESS elif line.command == "904": match.pop(0) + else: + break return SASLResult.FAILURE From 0edcbfa234a86163569fe0b2a403e46be0bb34f3 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sun, 19 Sep 2021 21:36:20 +0000 Subject: [PATCH 28/58] v0.4.4 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 17b2ccd..6f2743d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.3 +0.4.4 From 6c91ebc7ecce4fadf434bc54663b5006b029f8b8 Mon Sep 17 00:00:00 2001 From: jesopo Date: Mon, 29 Nov 2021 16:09:26 +0000 Subject: [PATCH 29/58] add ConnectionParams.from_hoststring("nick", "host:+port") --- ircrobots/params.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ircrobots/params.py b/ircrobots/params.py index fcbdbc2..27117e3 100644 --- a/ircrobots/params.py +++ b/ircrobots/params.py @@ -50,3 +50,20 @@ class ConnectionParams(object): alt_nicknames: List[str] = field(default_factory=list) autojoin: List[str] = field(default_factory=list) + + @staticmethod + def from_hoststring( + nickname: str, + hoststring: str + ) -> "ConnectionParams": + + host, _, port_s = hoststring.strip().partition(":") + + if port_s.startswith("+"): + tls = True + port_s = port_s.lstrip("+") or "6697" + elif not port_s: + tls = False + port_s = "6667" + + return ConnectionParams(nickname, host, int(port_s), tls) From ac4c144d58aaf7b70ad7248a385922fe6528dbcb Mon Sep 17 00:00:00 2001 From: jesopo Date: Mon, 29 Nov 2021 16:11:54 +0000 Subject: [PATCH 30/58] v0.4.5 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6f2743d..0bfccb0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.4 +0.4.5 From 05750f00d977e6254a395ed8f06d45532f685985 Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 9 Dec 2021 23:49:40 +0000 Subject: [PATCH 31/58] make sure 'tls' is defined --- ircrobots/params.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ircrobots/params.py b/ircrobots/params.py index 27117e3..9c8ea97 100644 --- a/ircrobots/params.py +++ b/ircrobots/params.py @@ -65,5 +65,7 @@ class ConnectionParams(object): elif not port_s: tls = False port_s = "6667" + else: + tls = False return ConnectionParams(nickname, host, int(port_s), tls) From 025fde97ee0f99e3c852a29f27d2f31a8f00a144 Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 9 Dec 2021 23:53:14 +0000 Subject: [PATCH 32/58] v0.4.6 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0bfccb0..ef52a64 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.5 +0.4.6 From 9ba5b2b90feaacb196369a0f82b649b8b27f5fe4 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 18 Dec 2021 16:48:01 +0000 Subject: [PATCH 33/58] add `transport` (ITCPTransport) param to bot.add_server --- ircrobots/bot.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ircrobots/bot.py b/ircrobots/bot.py index 3a29f7f..11090d0 100644 --- a/ircrobots/bot.py +++ b/ircrobots/bot.py @@ -6,7 +6,7 @@ from ircstates.server import ServerDisconnectedException from .server import ConnectionParams, Server from .transport import TCPTransport -from .interface import IBot, IServer +from .interface import IBot, IServer, ITCPTransport class Bot(IBot): def __init__(self): @@ -38,10 +38,13 @@ class Bot(IBot): del self.servers[server.name] await server.disconnect() - async def add_server(self, name: str, params: ConnectionParams) -> Server: + async def add_server(self, + name: str, + params: ConnectionParams, + transport: ITCPTransport = TCPTransport()) -> Server: server = self.create_server(name) self.servers[name] = server - await server.connect(TCPTransport(), params) + await server.connect(transport, params) await self._server_queue.put(server) return server From 3e18deef8646bd1fb3b62cfe72f8e6e6eb046419 Mon Sep 17 00:00:00 2001 From: jesopo Date: Fri, 7 Jan 2022 11:41:35 +0000 Subject: [PATCH 34/58] we don't support py3.6; support py3.9 --- .travis.yml | 2 +- requirements.txt | 1 - setup.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4311574..8b924d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ cache: pip python: - "3.7" - "3.8" - - "3.8-dev" + - "3.9" install: - pip3 install mypy -r requirements.txt script: diff --git a/requirements.txt b/requirements.txt index 9437d30..347b24f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ anyio ~=2.0.2 asyncio-rlock ~=0.1.0 asyncio-throttle ~=1.0.1 -dataclasses ~=0.6; python_version<"3.7" ircstates ~=0.11.10 async_stagger ~=0.3.0 async_timeout ~=3.0.1 diff --git a/setup.py b/setup.py index 7bc3e5c..b4b6b17 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ setup( "Operating System :: Microsoft :: Windows", "Topic :: Communications :: Chat :: Internet Relay Chat" ], - python_requires='>=3.6', + python_requires='>=3.7', install_requires=install_requires ) From fcd2f5b1b22b4abe99789e60be419b6a347e3780 Mon Sep 17 00:00:00 2001 From: jesopo Date: Fri, 7 Jan 2022 11:43:30 +0000 Subject: [PATCH 35/58] upgrade ircstates --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 347b24f..28480a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ anyio ~=2.0.2 asyncio-rlock ~=0.1.0 asyncio-throttle ~=1.0.1 -ircstates ~=0.11.10 +ircstates ~=0.11.11 async_stagger ~=0.3.0 async_timeout ~=3.0.1 From 66358f77e3156b02e3ee38d1bde4a185883492a0 Mon Sep 17 00:00:00 2001 From: jesopo Date: Fri, 7 Jan 2022 11:45:32 +0000 Subject: [PATCH 36/58] v0.4.7 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ef52a64..f905682 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.6 +0.4.7 From b7019d35c1d0ae258431c7b3a82402940478c279 Mon Sep 17 00:00:00 2001 From: jesopo Date: Fri, 7 Jan 2022 19:04:48 +0000 Subject: [PATCH 37/58] upgrade ircstates --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 28480a4..b988845 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ anyio ~=2.0.2 asyncio-rlock ~=0.1.0 asyncio-throttle ~=1.0.1 -ircstates ~=0.11.11 +ircstates ~=0.12.0 async_stagger ~=0.3.0 async_timeout ~=3.0.1 From 80b941fa53804c01704e698bc9baecec43658661 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sun, 16 Jan 2022 15:24:22 +0000 Subject: [PATCH 38/58] handle ERR_UNAVAILRESOURCE for prereg NICK failure too --- ircrobots/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ircrobots/server.py b/ircrobots/server.py index d916761..39d2e04 100644 --- a/ircrobots/server.py +++ b/ircrobots/server.py @@ -181,9 +181,9 @@ class Server(IServer): self._pending_who[0] == chan): self._pending_who.popleft() await self._next_who() - - elif (line.command in {ERR_NICKNAMEINUSE, ERR_ERRONEUSNICKNAME} and - not self.registered): + elif (line.command in { + ERR_NICKNAMEINUSE, ERR_ERRONEUSNICKNAME, ERR_UNAVAILRESOURCE + } and not self.registered): if self._alt_nicks: nick = self._alt_nicks.pop(0) await self.send(build("NICK", [nick])) From 8245a411c0407e36049a61c2932fd2ac59698a7e Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 20 Jan 2022 21:24:54 +0000 Subject: [PATCH 39/58] hmm no this isnt how you ask for cert validation apparently --- ircrobots/security.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/ircrobots/security.py b/ircrobots/security.py index 17d1b78..7b65236 100644 --- a/ircrobots/security.py +++ b/ircrobots/security.py @@ -1,13 +1,4 @@ import ssl def tls_context(verify: bool=True) -> ssl.SSLContext: - context = ssl.SSLContext(ssl.PROTOCOL_TLS) - context.options |= ssl.OP_NO_SSLv2 - context.options |= ssl.OP_NO_SSLv3 - context.options |= ssl.OP_NO_TLSv1 - context.load_default_certs() - - if verify: - context.verify_mode = ssl.CERT_REQUIRED - - return context + return ssl.create_default_context() From 0a5c7749659ec63b86d23c9bd0ca085047d20072 Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 20 Jan 2022 21:28:23 +0000 Subject: [PATCH 40/58] v0.5.0 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f905682..8f0916f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.7 +0.5.0 From 5b347f95c9ff4f7f63f2b2a15bf61652a0a7809c Mon Sep 17 00:00:00 2001 From: jesopo Date: Sun, 23 Jan 2022 16:52:27 +0000 Subject: [PATCH 41/58] combine params.tls and .tls_verify, support pinned certs --- examples/factoids.py | 4 ++-- examples/sasl.py | 1 - examples/simple.py | 2 +- ircrobots/interface.py | 10 +++++----- ircrobots/ircv3.py | 7 ++++--- ircrobots/params.py | 32 ++++++++++++++++++++++---------- ircrobots/security.py | 26 +++++++++++++++++++++++++- ircrobots/server.py | 5 ++--- ircrobots/transport.py | 32 ++++++++++++++++++++++++-------- 9 files changed, 85 insertions(+), 34 deletions(-) diff --git a/examples/factoids.py b/examples/factoids.py index bea0b4c..336dc2b 100644 --- a/examples/factoids.py +++ b/examples/factoids.py @@ -154,8 +154,8 @@ async def main(hostname: str, channel: str, nickname: str): params = ConnectionParams( nickname, hostname, - 6697, - tls=True) + 6697 + ) await bot.add_server("freenode", params) await bot.run() diff --git a/examples/sasl.py b/examples/sasl.py index e8e9818..97c81fa 100644 --- a/examples/sasl.py +++ b/examples/sasl.py @@ -23,7 +23,6 @@ async def main(): "MyNickname", host = "chat.freenode.invalid", port = 6697, - tls = True, sasl = sasl_params) await bot.add_server("freenode", params) diff --git a/examples/simple.py b/examples/simple.py index 54892b4..e47fc1b 100644 --- a/examples/simple.py +++ b/examples/simple.py @@ -25,7 +25,7 @@ class Bot(BaseBot): async def main(): bot = Bot() for name, host in SERVERS: - params = ConnectionParams("BitBotNewTest", host, 6697, True) + params = ConnectionParams("BitBotNewTest", host, 6697) await bot.add_server(name, params) await bot.run() diff --git a/ircrobots/interface.py b/ircrobots/interface.py index f680f5f..db66353 100644 --- a/ircrobots/interface.py +++ b/ircrobots/interface.py @@ -6,6 +6,7 @@ from ircstates import Server, Emit from irctokens import Line, Hostmask from .params import ConnectionParams, SASLParams, STSPolicy, ResumePolicy +from .security import TLS class ITCPReader(object): async def read(self, byte_count: int): @@ -24,11 +25,10 @@ class ITCPWriter(object): class ITCPTransport(object): async def connect(self, - hostname: str, - port: int, - tls: bool, - tls_verify: bool=True, - bindhost: Optional[str]=None + hostname: str, + port: int, + tls: Optional[TLS], + bindhost: Optional[str]=None ) -> Tuple[ITCPReader, ITCPWriter]: pass diff --git a/ircrobots/ircv3.py b/ircrobots/ircv3.py index b26ab8f..359ca12 100644 --- a/ircrobots/ircv3.py +++ b/ircrobots/ircv3.py @@ -8,6 +8,7 @@ from .contexts import ServerContext from .matching import Response, ANY from .interface import ICapability from .params import ConnectionParams, STSPolicy, ResumePolicy +from .security import TLS_VERIFYCHAIN class Capability(ICapability): def __init__(self, @@ -101,12 +102,12 @@ def _cap_dict(s: str) -> Dict[str, str]: return d async def sts_transmute(params: ConnectionParams): - if not params.sts is None and not params.tls: + if not params.sts is None and params.tls is None: now = time() since = (now-params.sts.created) if since <= params.sts.duration: params.port = params.sts.port - params.tls = True + params.tls = TLS_VERIFYCHAIN async def resume_transmute(params: ConnectionParams): if params.resume is not None: params.host = params.resume.address @@ -182,7 +183,7 @@ class CAPContext(ServerContext): if not params.tls: if "port" in sts_dict: params.port = int(sts_dict["port"]) - params.tls = True + params.tls = TLS_VERIFYCHAIN await self.server.bot.disconnect(self.server) await self.server.bot.add_server(self.server.name, params) diff --git a/ircrobots/params.py b/ircrobots/params.py index 9c8ea97..e52d6d6 100644 --- a/ircrobots/params.py +++ b/ircrobots/params.py @@ -1,6 +1,9 @@ +from re import compile as re_compile from typing import List, Optional from dataclasses import dataclass, field +from .security import TLS, TLS_NOVERIFY, TLS_VERIFYCHAIN + class SASLParams(object): mechanism: str @@ -28,19 +31,24 @@ class ResumePolicy(object): address: str token: str +RE_IPV6HOST = re_compile("\[([a-fA-F0-9:]+)\]") + +_TLS_TYPES = { + "+": TLS_VERIFYCHAIN, + "~": TLS_NOVERIFY +} @dataclass class ConnectionParams(object): nickname: str host: str port: int - tls: bool + tls: Optional[TLS] = TLS_VERIFYCHAIN username: Optional[str] = None realname: Optional[str] = None bindhost: Optional[str] = None password: Optional[str] = None - tls_verify: bool = True sasl: Optional[SASLParams] = None sts: Optional[STSPolicy] = None @@ -57,15 +65,19 @@ class ConnectionParams(object): hoststring: str ) -> "ConnectionParams": - host, _, port_s = hoststring.strip().partition(":") + ipv6host = RE_IPV6HOST.search(hoststring) + if ipv6host is not None and ipv6host.start() == 0: + host = ipv6host.group(1) + port_s = hoststring[ipv6host.end()+1:] + else: + host, _, port_s = hoststring.strip().partition(":") - if port_s.startswith("+"): - tls = True - port_s = port_s.lstrip("+") or "6697" - elif not port_s: - tls = False + tls_type: Optional[TLS] = None + if not port_s: port_s = "6667" else: - tls = False + tls_type = _TLS_TYPES.get(port_s[0], None) + if tls_type is not None: + port_s = port_s[1:] or "6697" - return ConnectionParams(nickname, host, int(port_s), tls) + return ConnectionParams(nickname, host, int(port_s), tls_type) diff --git a/ircrobots/security.py b/ircrobots/security.py index 7b65236..f10b700 100644 --- a/ircrobots/security.py +++ b/ircrobots/security.py @@ -1,4 +1,28 @@ import ssl +class TLS: + pass + +# tls without verification +class TLSNoVerify(TLS): + pass +TLS_NOVERIFY = TLSNoVerify() + +# verify via CAs +class TLSVerifyChain(TLS): + pass +TLS_VERIFYCHAIN = TLSVerifyChain() + +# verify by a pinned hash +class TLSVerifyHash(TLSNoVerify): + def __init__(self, sum: str): + self.sum = sum.lower() +class TLSVerifySHA512(TLSVerifyHash): + pass + def tls_context(verify: bool=True) -> ssl.SSLContext: - return ssl.create_default_context() + ctx = ssl.create_default_context() + if not verify: + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx diff --git a/ircrobots/server.py b/ircrobots/server.py index 39d2e04..d29946d 100644 --- a/ircrobots/server.py +++ b/ircrobots/server.py @@ -124,9 +124,8 @@ class Server(IServer): reader, writer = await transport.connect( params.host, params.port, - tls =params.tls, - tls_verify=params.tls_verify, - bindhost =params.bindhost) + tls =params.tls, + bindhost =params.bindhost) self._reader = reader self._writer = writer diff --git a/ircrobots/transport.py b/ircrobots/transport.py index 291409c..a7cb330 100644 --- a/ircrobots/transport.py +++ b/ircrobots/transport.py @@ -1,10 +1,12 @@ +from hashlib import sha512 from ssl import SSLContext from typing import Optional, Tuple from asyncio import StreamReader, StreamWriter from async_stagger import open_connection from .interface import ITCPTransport, ITCPReader, ITCPWriter -from .security import tls_context +from .security import (tls_context, TLS, TLSNoVerify, TLSVerifyHash, + TLSVerifySHA512) class TCPReader(ITCPReader): def __init__(self, reader: StreamReader): @@ -32,16 +34,15 @@ class TCPWriter(ITCPWriter): class TCPTransport(ITCPTransport): async def connect(self, - hostname: str, - port: int, - tls: bool, - tls_verify: bool=True, - bindhost: Optional[str]=None + hostname: str, + port: int, + tls: Optional[TLS], + bindhost: Optional[str]=None ) -> Tuple[ITCPReader, ITCPWriter]: cur_ssl: Optional[SSLContext] = None - if tls: - cur_ssl = tls_context(tls_verify) + if tls is not None: + cur_ssl = tls_context(not isinstance(tls, TLSNoVerify)) local_addr: Optional[Tuple[str, int]] = None if not bindhost is None: @@ -55,5 +56,20 @@ class TCPTransport(ITCPTransport): server_hostname=server_hostname, ssl =cur_ssl, local_addr =local_addr) + + if isinstance(tls, TLSVerifyHash): + cert: bytes = writer.transport.get_extra_info( + "ssl_object" + ).getpeercert(True) + if isinstance(tls, TLSVerifySHA512): + sum = sha512(cert).hexdigest() + else: + raise ValueError(f"unknown hash pinning {type(tls)}") + + if not sum == tls.sum: + raise ValueError( + f"pinned hash for {hostname} does not match ({sum})" + ) + return (TCPReader(reader), TCPWriter(writer)) From 0ce3b9b0b080f03948dc8e2d2cfa1394640142b7 Mon Sep 17 00:00:00 2001 From: jesopo Date: Mon, 24 Jan 2022 10:01:51 +0000 Subject: [PATCH 42/58] v0.6.0 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8f0916f..a918a2a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.0 +0.6.0 From 20c4f8f98cca3082aac3d329c65e1c924a412412 Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 19 Feb 2022 13:49:21 +0000 Subject: [PATCH 43/58] upgrade async-timeout to v4.0.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b988845..5405dbb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ asyncio-rlock ~=0.1.0 asyncio-throttle ~=1.0.1 ircstates ~=0.12.0 async_stagger ~=0.3.0 -async_timeout ~=3.0.1 +async_timeout ~=4.0.2 From 63025af31106605593ec94cc193f5f141d5b5faa Mon Sep 17 00:00:00 2001 From: jesopo Date: Sat, 19 Feb 2022 13:51:05 +0000 Subject: [PATCH 44/58] v0.6.1 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a918a2a..ee6cdce 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.0 +0.6.1 From 0435404ec3bacd1eb44b83c3a03f7734c7e61a01 Mon Sep 17 00:00:00 2001 From: alicetries <92898519+alicetries@users.noreply.github.com> Date: Mon, 28 Mar 2022 23:38:48 +0100 Subject: [PATCH 45/58] Small tweak to how repr() of Formatless() displays --- ircrobots/matching/params.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ircrobots/matching/params.py b/ircrobots/matching/params.py index c038db4..ec2e8ff 100644 --- a/ircrobots/matching/params.py +++ b/ircrobots/matching/params.py @@ -73,8 +73,7 @@ class Formatless(IMatchResponseParam): def __init__(self, value: TYPE_MAYBELIT_VALUE): self._value = _assure_lit(value) def __repr__(self) -> str: - brepr = super().__repr__() - return f"Formatless({brepr})" + return f"Formatless({self._value!r})" def match(self, server: IServer, arg: str) -> bool: strip = formatting.strip(arg) return self._value.match(server, strip) From 9a2f2156fe08f89e714f8a16122b64f518a92f33 Mon Sep 17 00:00:00 2001 From: jesopo Date: Mon, 6 Feb 2023 19:42:27 +0000 Subject: [PATCH 46/58] support specifying tls client keypair --- ircrobots/__init__.py | 1 + ircrobots/security.py | 5 ++++- ircrobots/transport.py | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ircrobots/__init__.py b/ircrobots/__init__.py index 5b798ed..c033b8f 100644 --- a/ircrobots/__init__.py +++ b/ircrobots/__init__.py @@ -3,3 +3,4 @@ from .server import Server from .params import (ConnectionParams, SASLUserPass, SASLExternal, SASLSCRAM, STSPolicy, ResumePolicy) from .ircv3 import Capability +from .security import TLS diff --git a/ircrobots/security.py b/ircrobots/security.py index f10b700..373c34f 100644 --- a/ircrobots/security.py +++ b/ircrobots/security.py @@ -1,7 +1,10 @@ import ssl +from dataclasses import dataclass +from typing import Optional, Tuple +@dataclass class TLS: - pass + client_keypair: Optional[Tuple[str, str]] = None # tls without verification class TLSNoVerify(TLS): diff --git a/ircrobots/transport.py b/ircrobots/transport.py index a7cb330..3a43cb3 100644 --- a/ircrobots/transport.py +++ b/ircrobots/transport.py @@ -43,6 +43,9 @@ class TCPTransport(ITCPTransport): cur_ssl: Optional[SSLContext] = None if tls is not None: cur_ssl = tls_context(not isinstance(tls, TLSNoVerify)) + if tls.client_keypair is not None: + (client_cert, client_key) = tls.client_keypair + cur_ssl.load_cert_chain(client_cert, keyfile=client_key) local_addr: Optional[Tuple[str, int]] = None if not bindhost is None: From 7bb4c3d069666ef18e4591389f1aa1a1bf47821e Mon Sep 17 00:00:00 2001 From: jesopo Date: Mon, 6 Feb 2023 19:43:14 +0000 Subject: [PATCH 47/58] v0.6.2 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ee6cdce..b616048 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1 +0.6.2 From b04a0e0136ff04bbfc7f222f31f247f63d8ff12a Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 6 Jul 2023 00:35:13 +0000 Subject: [PATCH 48/58] python no longer likes having mutables in non-default_factory --- ircrobots/params.py | 10 +++++----- ircrobots/security.py | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ircrobots/params.py b/ircrobots/params.py index e52d6d6..e2699fe 100644 --- a/ircrobots/params.py +++ b/ircrobots/params.py @@ -2,7 +2,7 @@ from re import compile as re_compile from typing import List, Optional from dataclasses import dataclass, field -from .security import TLS, TLS_NOVERIFY, TLS_VERIFYCHAIN +from .security import TLS, TLSNoVerify, TLSVerifyChain class SASLParams(object): mechanism: str @@ -34,15 +34,15 @@ class ResumePolicy(object): RE_IPV6HOST = re_compile("\[([a-fA-F0-9:]+)\]") _TLS_TYPES = { - "+": TLS_VERIFYCHAIN, - "~": TLS_NOVERIFY + "+": TLSVerifyChain, + "~": TLSNoVerify, } @dataclass class ConnectionParams(object): nickname: str host: str port: int - tls: Optional[TLS] = TLS_VERIFYCHAIN + tls: Optional[TLS] = field(default_factory=TLSVerifyChain) username: Optional[str] = None realname: Optional[str] = None @@ -76,7 +76,7 @@ class ConnectionParams(object): if not port_s: port_s = "6667" else: - tls_type = _TLS_TYPES.get(port_s[0], None) + tls_type = _TLS_TYPES.get(port_s[0], lambda: None)() if tls_type is not None: port_s = port_s[1:] or "6697" diff --git a/ircrobots/security.py b/ircrobots/security.py index 373c34f..96c9f5c 100644 --- a/ircrobots/security.py +++ b/ircrobots/security.py @@ -9,12 +9,10 @@ class TLS: # tls without verification class TLSNoVerify(TLS): pass -TLS_NOVERIFY = TLSNoVerify() # verify via CAs class TLSVerifyChain(TLS): pass -TLS_VERIFYCHAIN = TLSVerifyChain() # verify by a pinned hash class TLSVerifyHash(TLSNoVerify): From 422a9a93c14a22ad749807b99f55af327f1c4247 Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 6 Jul 2023 00:35:44 +0000 Subject: [PATCH 49/58] v0.6.3 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b616048..844f6a9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.2 +0.6.3 From 81fa77cf29acf773a8a25549a17cf48fa6e5c6d6 Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 6 Jul 2023 00:44:13 +0000 Subject: [PATCH 50/58] missed some TLS_ uses --- ircrobots/ircv3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ircrobots/ircv3.py b/ircrobots/ircv3.py index 359ca12..3795237 100644 --- a/ircrobots/ircv3.py +++ b/ircrobots/ircv3.py @@ -8,7 +8,7 @@ from .contexts import ServerContext from .matching import Response, ANY from .interface import ICapability from .params import ConnectionParams, STSPolicy, ResumePolicy -from .security import TLS_VERIFYCHAIN +from .security import TLSVerifyChain class Capability(ICapability): def __init__(self, @@ -107,7 +107,7 @@ async def sts_transmute(params: ConnectionParams): since = (now-params.sts.created) if since <= params.sts.duration: params.port = params.sts.port - params.tls = TLS_VERIFYCHAIN + params.tls = TLSVerifyChain() async def resume_transmute(params: ConnectionParams): if params.resume is not None: params.host = params.resume.address @@ -183,7 +183,7 @@ class CAPContext(ServerContext): if not params.tls: if "port" in sts_dict: params.port = int(sts_dict["port"]) - params.tls = TLS_VERIFYCHAIN + params.tls = TLSVerifyChain() await self.server.bot.disconnect(self.server) await self.server.bot.add_server(self.server.name, params) From a1a459c13ecc76c60bda5a48f4d2f2f81939f20a Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 6 Jul 2023 00:44:25 +0000 Subject: [PATCH 51/58] v0.6.4 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 844f6a9..d2b13eb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.3 +0.6.4 From cf2e69a9e2a409de3e1516f93b320d5eb101c875 Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 6 Jul 2023 00:56:45 +0000 Subject: [PATCH 52/58] asyncio.wait(..) now requires Tasks --- ircrobots/server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ircrobots/server.py b/ircrobots/server.py index d29946d..64ff5e9 100644 --- a/ircrobots/server.py +++ b/ircrobots/server.py @@ -287,9 +287,10 @@ class Server(IServer): if not self._process_queue: async with self._read_lwork: - read_aw = self._read_line(PING_TIMEOUT) + read_aw = asyncio.create_task(self._read_line(PING_TIMEOUT)) + wait_aw = asyncio.create_task(self._wait_for.wait()) dones, notdones = await asyncio.wait( - [read_aw, self._wait_for.wait()], + [read_aw, wait_aw], return_when=asyncio.FIRST_COMPLETED ) self._wait_for.clear() From f2ba48a5820ef0392960d3245cac7964cf4e5bea Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 6 Jul 2023 00:57:08 +0000 Subject: [PATCH 53/58] v0.6.5 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d2b13eb..ef5e445 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.4 +0.6.5 From e3c91a50e172f3e22a0206860024742a21996ecc Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 17 Aug 2023 22:47:57 +0000 Subject: [PATCH 54/58] update ircstates --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5405dbb..80ea8ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ anyio ~=2.0.2 asyncio-rlock ~=0.1.0 asyncio-throttle ~=1.0.1 -ircstates ~=0.12.0 +ircstates ~=0.12.1 async_stagger ~=0.3.0 async_timeout ~=4.0.2 From 7c9a144124b69be57d09f8206b73c61f0e6a042e Mon Sep 17 00:00:00 2001 From: jesopo Date: Thu, 17 Aug 2023 22:48:21 +0000 Subject: [PATCH 55/58] v0.6.6 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ef5e445..05e8a45 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.5 +0.6.6 From c0319afec1b63b546277b61fa5b6b6976102269a Mon Sep 17 00:00:00 2001 From: jesopo Date: Tue, 17 Sep 2024 10:20:43 +0000 Subject: [PATCH 56/58] bump ircstates to v0.13.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 80ea8ad..4193582 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ anyio ~=2.0.2 asyncio-rlock ~=0.1.0 asyncio-throttle ~=1.0.1 -ircstates ~=0.12.1 +ircstates ~=0.13.0 async_stagger ~=0.3.0 async_timeout ~=4.0.2 From eaa54db664e65fbf63415510ea395be9cd639182 Mon Sep 17 00:00:00 2001 From: jesopo Date: Tue, 17 Sep 2024 10:21:00 +0000 Subject: [PATCH 57/58] v0.7.0 release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 05e8a45..faef31a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.6 +0.7.0 From 63aa84b40450bd534fc232eee10e8088028c9f6d Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Fri, 20 Sep 2024 21:01:21 +0200 Subject: [PATCH 58/58] =?UTF-8?q?fix:=20spawn=20=E2=86=92=20start=5Fsoon?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://anyio.readthedocs.io/en/stable/versionhistory.html v3.0.0 Signed-off-by: Raito Bezarius --- ircrobots/bot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ircrobots/bot.py b/ircrobots/bot.py index 11090d0..809fb5e 100644 --- a/ircrobots/bot.py +++ b/ircrobots/bot.py @@ -51,8 +51,8 @@ class Bot(IBot): async def _run_server(self, server: Server): try: async with anyio.create_task_group() as tg: - await tg.spawn(server._read_lines) - await tg.spawn(server._send_lines) + tg.start_soon(server._read_lines) + tg.start_soon(server._send_lines) except ServerDisconnectedException: server.disconnected = True @@ -62,4 +62,4 @@ class Bot(IBot): async with anyio.create_task_group() as tg: while not tg.cancel_scope.cancel_called: server = await self._server_queue.get() - await tg.spawn(self._run_server, server) + tg.start_soon(self._run_server, server)