From 09621f53716240c6a48cac7f09193a43b0ad305b Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 26 Jan 2020 01:18:34 +0000 Subject: [PATCH 1/8] refactor(lisp/dns): Split package into multiple files Adds a package definition file and moves the current client into client.lisp Note that the client is not working at all at this commit as this is a work-in-progress snapshot. --- lisp/dns/client.lisp | 173 +++++++++++++++++++++++++++++++++++++++++ lisp/dns/default.nix | 7 +- lisp/dns/package.lisp | 4 + lisp/dns/resolver.lisp | 29 ------- 4 files changed, 182 insertions(+), 31 deletions(-) create mode 100644 lisp/dns/client.lisp create mode 100644 lisp/dns/package.lisp delete mode 100644 lisp/dns/resolver.lisp diff --git a/lisp/dns/client.lisp b/lisp/dns/client.lisp new file mode 100644 index 000000000..0f355589d --- /dev/null +++ b/lisp/dns/client.lisp @@ -0,0 +1,173 @@ +;; Implementation of a DoH-client, see RFC 8484 (DNS Queries over +;; HTTPS (DoH)) + +(in-package #:dns) + +;; The DoH client is configured with a URI Template [RFC6570] +(defvar *doh-base-url* "https://dns.google/dns-query" + "Base URL of the service providing DNS-over-HTTP(S). Defaults to the + Google-hosted API.") + +(defun lookup-generic (name type) + (multiple-value-bind (stream) + (drakma:http-request *doh-base-url* + :decode-content t + :want-stream t + :parameters `(("type" . ,type) + ("name" . ,name) + ("ct" . "application/dns-message"))) + (read-binary 'dns-message stream))) + +(defun lookup-txt (name) + "Look up the TXT records at NAME." + (lookup-generic name "TXT")) + +(defun lookup-mx (name) + "Look up the MX records at NAME." + (lookup-generic name "MX")) + + +;; The URI Template defined in this document is processed without any +;; variables when the HTTP method is POST. When the HTTP method is GET, +;; the single variable "dns" is defined as the content of the DNS +;; request (as described in Section 6), encoded with base64url +;; [RFC4648]. + +;; When using the POST method, the DNS query is included as the message +;; body of the HTTP request, and the Content-Type request header field +;; indicates the media type of the message. POSTed requests are +;; generally smaller than their GET equivalents. + +;; Using the GET method is friendlier to many HTTP cache +;; implementations. + +;; The DoH client SHOULD include an HTTP Accept request header field to +;; indicate what type of content can be understood in response. +;; Irrespective of the value of the Accept request header field, the +;; client MUST be prepared to process "application/dns-message" (as +;; described in Section 6) responses but MAY also process other DNS- +;; related media types it receives. + +;; In order to maximize HTTP cache friendliness, DoH clients using media +;; formats that include the ID field from the DNS message header, such +;; as "application/dns-message", SHOULD use a DNS ID of 0 in every DNS +;; request. HTTP correlates the request and response, thus eliminating +;; the need for the ID in a media type such as "application/dns- +;; message". The use of a varying DNS ID can cause semantically +;; equivalent DNS queries to be cached separately. + +;; DoH clients can use HTTP/2 padding and compression [RFC7540] in the +;; same way that other HTTP/2 clients use (or don't use) them. + +;; 4.1.1. HTTP Request Examples + +;; These examples use HTTP/2-style formatting from [RFC7540]. + +;; These examples use a DoH service with a URI Template of +;; "https://dnsserver.example.net/dns-query{?dns}" to resolve IN A +;; records. + +;; The requests are represented as bodies with media type "application/ +;; dns-message". + +;; The first example request uses GET to request "www.example.com". + +;; :method = GET +;; :scheme = https +;; :authority = dnsserver.example.net +;; :path = /dns-query?dns=AAABAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB +;; accept = application/dns-message + +;; Finally, a GET-based query for "a.62characterlabel-makes-base64url- +;; distinct-from-standard-base64.example.com" is shown as an example to +;; emphasize that the encoding alphabet of base64url is different than +;; regular base64 and that padding is omitted. + +;; The only response type defined in this document is "application/dns- +;; message", but it is possible that other response formats will be +;; defined in the future. A DoH server MUST be able to process +;; "application/dns-message" request messages. + +;; Each DNS request-response pair is mapped to one HTTP exchange. + +;; DNS response codes indicate either success or failure for the DNS +;; query. A successful HTTP response with a 2xx status code (see +;; Section 6.3 of [RFC7231]) is used for any valid DNS response, + +;; HTTP responses with non-successful HTTP status codes do not contain +;; replies to the original DNS question in the HTTP request. DoH +;; clients need to use the same semantic processing of non-successful +;; HTTP status codes as other HTTP clients. + +;; 4.2.2. HTTP Response Example + +;; This is an example response for a query for the IN AAAA records for +;; "www.example.com" with recursion turned on. The response bears one +;; answer record with an address of 2001:db8:abcd:12:1:2:3:4 and a TTL +;; of 3709 seconds. + +;; :status = 200 +;; content-type = application/dns-message +;; content-length = 61 +;; cache-control = max-age=3709 + +;; <61 bytes represented by the following hex encoding> +;; 00 00 81 80 00 01 00 01 00 00 00 00 03 77 77 77 +;; 07 65 78 61 6d 70 6c 65 03 63 6f 6d 00 00 1c 00 +;; 01 c0 0c 00 1c 00 01 00 00 0e 7d 00 10 20 01 0d +;; b8 ab cd 00 12 00 01 00 02 00 03 00 04 + +;; This protocol MUST be used with the https URI scheme [RFC7230]. + +;; In particular, DoH servers SHOULD assign an explicit HTTP freshness +;; lifetime (see Section 4.2 of [RFC7234]) so that the DoH client is +;; more likely to use fresh DNS data. This requirement is due to HTTP +;; caches being able to assign their own heuristic freshness (such as +;; that described in Section 4.2.2 of [RFC7234]), which would take +;; control of the cache contents out of the hands of the DoH server. + + +;; The assigned freshness lifetime of a DoH HTTP response MUST be less +;; than or equal to the smallest TTL in the Answer section of the DNS +;; response. A freshness lifetime equal to the smallest TTL in the +;; Answer section is RECOMMENDED. For example, if a HTTP response +;; carries three RRsets with TTLs of 30, 600, and 300, the HTTP +;; freshness lifetime should be 30 seconds (which could be specified as +;; "Cache-Control: max-age=30"). This requirement helps prevent expired +;; RRsets in messages in an HTTP cache from unintentionally being +;; served. + +;; If the DNS response has no records in the Answer section, and the DNS +;; response has an SOA record in the Authority section, the response +;; freshness lifetime MUST NOT be greater than the MINIMUM field from +;; that SOA record (see [RFC2308]). + +;; DoH clients MUST account for the Age response header field's value +;; [RFC7234] when calculating the DNS TTL of a response. For example, +;; if an RRset is received with a DNS TTL of 600, but the Age header +;; field indicates that the response has been cached for 250 seconds, +;; the remaining lifetime of the RRset is 350 seconds. This requirement +;; applies to both DoH client HTTP caches and DoH client DNS caches. + +;; Those features were introduced to HTTP in HTTP/2 [RFC7540]. +;; Earlier versions of HTTP are capable of conveying the semantic +;; requirements of DoH but may result in very poor performance. + +;; In order to maximize interoperability, DoH clients and DoH servers +;; MUST support the "application/dns-message" media type. + +;; The data payload for the "application/dns-message" media type is a +;; single message of the DNS on-the-wire format defined in Section 4.2.1 +;; of [RFC1035], which in turn refers to the full wire format defined in +;; Section 4.1 of that RFC. + +;; This media type restricts the maximum size of the DNS message to +;; 65535 bytes. + +;; When using the GET method, the data payload for this media type MUST +;; be encoded with base64url [RFC4648] and then provided as a variable +;; named "dns" to the URI Template expansion. Padding characters for +;; base64url MUST NOT be included. + +;; When using the POST method, the data payload for this media type MUST +;; NOT be encoded and is used directly as the HTTP message body. diff --git a/lisp/dns/default.nix b/lisp/dns/default.nix index c41b02f97..134540b2b 100644 --- a/lisp/dns/default.nix +++ b/lisp/dns/default.nix @@ -4,12 +4,15 @@ pkgs.nix.buildLisp.library { name = "dns"; deps = with pkgs.third_party.lisp; [ + drakma + lisp-binary + iterate alexandria cl-json - drakma ]; srcs = [ - ./resolver.lisp + ./package.lisp + ./client.lisp ]; } diff --git a/lisp/dns/package.lisp b/lisp/dns/package.lisp new file mode 100644 index 000000000..639d9994a --- /dev/null +++ b/lisp/dns/package.lisp @@ -0,0 +1,4 @@ +(defpackage #:dns + (:documentation "Simple DNS resolver in Common Lisp") + (:use #:cl #:iterate #:lisp-binary) + (:export #:lookup-txt #:lookup-mx)) diff --git a/lisp/dns/resolver.lisp b/lisp/dns/resolver.lisp deleted file mode 100644 index 774be525c..000000000 --- a/lisp/dns/resolver.lisp +++ /dev/null @@ -1,29 +0,0 @@ -;; Initial implementation is a simple client for -;; https://developers.google.com/speed/public-dns/docs/doh/json - -(defpackage #:dns - (:documentation "Simple DNS resolver in Common Lisp") - (:use #:cl) - (:export #:lookup-txt #:lookup-mx)) - -(defvar *doh-base-url* "https://dns.google/resolve" - "Base URL of the service providing DNS-over-HTTP(S). Defaults to the - Google-hosted API.") - -(defun lookup-generic (name type) - (multiple-value-bind (body) - (drakma:http-request *doh-base-url* - :decode-content t - :want-stream t - :parameters `(("type" . ,type) - ("name" . ,name) - ("ct" . "application/x-javascript"))) - (cl-json:decode-json body))) - -(defun lookup-txt (name) - "Look up the TXT records at NAME." - (lookup-generic name "TXT")) - -(defun lookup-mx (name) - "Look up the MX records at NAME." - (lookup-generic name "MX")) From 7ef14db936d98a1ae3def2d2560ee6373ea1a107 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 26 Jan 2020 01:19:12 +0000 Subject: [PATCH 2/8] feat(lisp/dns): Check in initial DNS message implementation This uses lisp-binary to define serialisation types for the DNS messages defined by RFC 1035. Currently the compression scheme used for QNAMEs is not supported, hence deserialisation of even simple records fails after the header and question sections are read. --- lisp/dns/message.lisp | 931 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 931 insertions(+) create mode 100644 lisp/dns/message.lisp diff --git a/lisp/dns/message.lisp b/lisp/dns/message.lisp new file mode 100644 index 000000000..9e283ea5c --- /dev/null +++ b/lisp/dns/message.lisp @@ -0,0 +1,931 @@ +(in-package :dns) + +;; 3. DOMAIN NAME SPACE AND RR DEFINITIONS 10 +;; 3.1. Name space definitions 10 +;; 3.2. RR definitions 11 +;; 3.2.1. Format 11 +;; 3.2.2. TYPE values 12 +;; 3.2.3. QTYPE values 12 +;; 3.2.4. CLASS values 13 +;; 3.2.5. QCLASS values 13 +;; 3.3. Standard RRs 13 +;; 3.3.1. CNAME RDATA format 14 +;; 3.3.2. HINFO RDATA format 14 +;; 3.3.3. MB RDATA format (EXPERIMENTAL) 14 +;; 3.3.4. MD RDATA format (Obsolete) 15 +;; 3.3.5. MF RDATA format (Obsolete) 15 +;; 3.3.6. MG RDATA format (EXPERIMENTAL) 16 +;; 3.3.7. MINFO RDATA format (EXPERIMENTAL) 16 +;; 3.3.8. MR RDATA format (EXPERIMENTAL) 17 +;; 3.3.9. MX RDATA format 17 +;; 3.3.10. NULL RDATA format (EXPERIMENTAL) 17 +;; 3.3.11. NS RDATA format 18 +;; 3.3.12. PTR RDATA format 18 +;; 3.3.13. SOA RDATA format 19 +;; 3.3.14. TXT RDATA format 20 +;; 3.4. ARPA Internet specific RRs 20 +;; 3.4.1. A RDATA format 20 +;; 3.4.2. WKS RDATA format 21 +;; 3.5. IN-ADDR.ARPA domain 22 +;; 3.6. Defining new types, classes, and special namespaces 24 +;; 4. MESSAGES 25 +;; 4.1. Format 25 +;; 4.1.1. Header section format 26 +;; 4.1.2. Question section format 28 +;; 4.1.3. Resource record format 29 +;; 4.1.4. Message compression 30 +;; 4.2. Transport 32 +;; 4.2.1. UDP usage 32 +;; 4.2.2. TCP usage 32 +;; 5. MASTER FILES 33 +;; 5.1. Format 33 +;; 5.2. Use of master files to define zones 35 +;; 5.3. Master file example 36 +;; 6. NAME SERVER IMPLEMENTATION 37 +;; 6.1. Architecture 37 +;; 6.1.1. Control 37 +;; 6.1.2. Database 37 +;; 6.1.3. Time 39 +;; 6.2. Standard query processing 39 +;; 6.3. Zone refresh and reload processing 39 +;; 6.4. Inverse queries (Optional) 40 +;; 6.4.1. The contents of inverse queries and responses 40 +;; 6.4.2. Inverse query and response example 41 +;; 6.4.3. Inverse query processing 42 +;; 6.5. Completion queries and responses 42 +;; 7. RESOLVER IMPLEMENTATION 43 +;; 7.1. Transforming a user request into a query 43 +;; 7.2. Sending the queries 44 +;; 7.3. Processing responses 46 +;; 7.4. Using the cache 47 +;; 8. MAIL SUPPORT 47 +;; 8.1. Mail exchange binding 48 +;; 8.2. Mailbox binding (Experimental) 48 +;; 9. REFERENCES and BIBLIOGRAPHY 50 +;; Index 54 + +;; 2.3.4. Size limits +;; Various objects and parameters in the DNS have size limits. They are +;; listed below. Some could be easily changed, others are more +;; fundamental. +;; labels 63 octets or less +;; names 255 octets or less +;; TTL positive values of a signed 32 bit number. +;; UDP messages 512 octets or less + +;; 3. DOMAIN NAME SPACE AND RR DEFINITIONS + +;; Domain names in messages are expressed in terms of a sequence of labels. +;; Each label is represented as a one octet length field followed by that +;; number of octets. Since every domain name ends with the null label of +;; the root, a domain name is terminated by a length byte of zero. The +;; high order two bits of every length octet must be zero, and the +;; remaining six bits of the length field limit the label to 63 octets or +;; less. + +;; To simplify implementations, the total length of a domain name (i.e., +;; label octets and label length octets) is restricted to 255 octets or +;; less. + +;; Although labels can contain any 8 bit values in octets that make up a +;; label, it is strongly recommended that labels follow the preferred +;; syntax described elsewhere in this memo, which is compatible with +;; existing host naming conventions. Name servers and resolvers must +;; compare labels in a case-insensitive manner (i.e., A=a), assuming ASCII +;; with zero parity. Non-alphabetic codes must match exactly. + +;; 3.2. RR definitions + +;; 3.2.1. Format + +;; All RRs have the same top level format shown below: + +;; 1 1 1 1 1 1 +;; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | | +;; / / +;; / NAME / +;; | | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | TYPE | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | CLASS | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | TTL | +;; | | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | RDLENGTH | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--| +;; / RDATA / +;; / / +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + + +;; where: + +;; NAME an owner name, i.e., the name of the node to which this +;; resource record pertains. + +;; TYPE two octets containing one of the RR TYPE codes. + +;; CLASS two octets containing one of the RR CLASS codes. + +;; TTL a 32 bit signed integer that specifies the time interval +;; that the resource record may be cached before the source +;; of the information should again be consulted. Zero +;; values are interpreted to mean that the RR can only be +;; used for the transaction in progress, and should not be +;; cached. For example, SOA records are always distributed +;; with a zero TTL to prohibit caching. Zero values can +;; also be used for extremely volatile data. + +;; RDLENGTH an unsigned 16 bit integer that specifies the length in +;; octets of the RDATA field. + +;; RDATA a variable length string of octets that describes the +;; resource. The format of this information varies +;; according to the TYPE and CLASS of the resource record. + +;; 3.2.2. TYPE values + +;; TYPE fields are used in resource records. Note that these types are a +;; subset of QTYPEs. + +;; TYPE value and meaning + +;; A 1 a host address + +;; NS 2 an authoritative name server + +;; MD 3 a mail destination (Obsolete - use MX) + +;; MF 4 a mail forwarder (Obsolete - use MX) + +;; CNAME 5 the canonical name for an alias + +;; SOA 6 marks the start of a zone of authority + +;; MB 7 a mailbox domain name (EXPERIMENTAL) + +;; MG 8 a mail group member (EXPERIMENTAL) + +;; MR 9 a mail rename domain name (EXPERIMENTAL) + +;; NULL 10 a null RR (EXPERIMENTAL) + +;; WKS 11 a well known service description + +;; PTR 12 a domain name pointer + +;; HINFO 13 host information + +;; MINFO 14 mailbox or mail list information + +;; MX 15 mail exchange + +;; TXT 16 text strings + +;; 3.2.3. QTYPE values + +;; QTYPE fields appear in the question part of a query. QTYPES are a +;; superset of TYPEs, hence all TYPEs are valid QTYPEs. In addition, the +;; following QTYPEs are defined: + +;; AXFR 252 A request for a transfer of an entire zone + +;; MAILB 253 A request for mailbox-related records (MB, MG or MR) + +;; MAILA 254 A request for mail agent RRs (Obsolete - see MX) + +;; * 255 A request for all records + +;; 3.2.4. CLASS values + +;; CLASS fields appear in resource records. The following CLASS mnemonics +;; and values are defined: + +;; IN 1 the Internet + +;; CS 2 the CSNET class (Obsolete - used only for examples in +;; some obsolete RFCs) + +;; CH 3 the CHAOS class + +;; HS 4 Hesiod [Dyer 87] + +;; 3.2.5. QCLASS values + +;; QCLASS fields appear in the question section of a query. QCLASS values +;; are a superset of CLASS values; every CLASS is a valid QCLASS. In +;; addition to CLASS values, the following QCLASSes are defined: + +;; * 255 any class + +;; 3.3. Standard RRs + +;; The following RR definitions are expected to occur, at least +;; potentially, in all classes. In particular, NS, SOA, CNAME, and PTR +;; will be used in all classes, and have the same format in all classes. +;; Because their RDATA format is known, all domain names in the RDATA +;; section of these RRs may be compressed. + +;; is a domain name represented as a series of labels, and +;; terminated by a label with zero length. is a single +;; length octet followed by that number of characters. +;; is treated as binary information, and can be up to 256 characters in +;; length (including the length octet). + +;; 3.3.1. CNAME RDATA format + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; / CNAME / +;; / / +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; where: + +;; CNAME A which specifies the canonical or primary +;; name for the owner. The owner name is an alias. + +;; CNAME RRs cause no additional section processing, but name servers may +;; choose to restart the query at the canonical name in certain cases. See +;; the description of name server logic in [RFC-1034] for details. + + +;; 3.3.11. NS RDATA format + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; / NSDNAME / +;; / / +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; where: + +;; NSDNAME A which specifies a host which should be +;; authoritative for the specified class and domain. + +;; NS records cause both the usual additional section processing to locate +;; a type A record, and, when used in a referral, a special search of the +;; zone in which they reside for glue information. + +;; The NS RR states that the named host should be expected to have a zone +;; starting at owner name of the specified class. Note that the class may +;; not indicate the protocol family which should be used to communicate +;; with the host, although it is typically a strong hint. For example, +;; hosts which are name servers for either Internet (IN) or Hesiod (HS) +;; class information are normally queried using IN class protocols. + +;; 3.3.12. PTR RDATA format + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; / PTRDNAME / +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; where: + +;; PTRDNAME A which points to some location in the +;; domain name space. + +;; PTR records cause no additional section processing. These RRs are used +;; in special domains to point to some other location in the domain space. +;; These records are simple data, and don't imply any special processing +;; similar to that performed by CNAME, which identifies aliases. See the +;; description of the IN-ADDR.ARPA domain for an example. + +;; 3.3.13. SOA RDATA format + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; / MNAME / +;; / / +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; / RNAME / +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | SERIAL | +;; | | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | REFRESH | +;; | | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | RETRY | +;; | | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | EXPIRE | +;; | | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | MINIMUM | +;; | | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; where: + +;; MNAME The of the name server that was the +;; original or primary source of data for this zone. + +;; RNAME A which specifies the mailbox of the +;; person responsible for this zone. + +;; SERIAL The unsigned 32 bit version number of the original copy +;; of the zone. Zone transfers preserve this value. This +;; value wraps and should be compared using sequence space +;; arithmetic. + +;; REFRESH A 32 bit time interval before the zone should be +;; refreshed. + +;; RETRY A 32 bit time interval that should elapse before a +;; failed refresh should be retried. + +;; EXPIRE A 32 bit time value that specifies the upper limit on +;; the time interval that can elapse before the zone is no +;; longer authoritative. + +;; MINIMUM The unsigned 32 bit minimum TTL field that should be +;; exported with any RR from this zone. + +;; SOA records cause no additional section processing. + +;; All times are in units of seconds. + +;; Most of these fields are pertinent only for name server maintenance +;; operations. However, MINIMUM is used in all query operations that +;; retrieve RRs from a zone. Whenever a RR is sent in a response to a +;; query, the TTL field is set to the maximum of the TTL field from the RR +;; and the MINIMUM field in the appropriate SOA. Thus MINIMUM is a lower +;; bound on the TTL field for all RRs in a zone. Note that this use of +;; MINIMUM should occur when the RRs are copied into the response and not +;; when the zone is loaded from a master file or via a zone transfer. The +;; reason for this provison is to allow future dynamic update facilities to +;; change the SOA RR with known semantics. + + +;; 3.3.14. TXT RDATA format + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; / TXT-DATA / +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; where: + +;; TXT-DATA One or more s. + +;; TXT RRs are used to hold descriptive text. The semantics of the text +;; depends on the domain where it is found. + +;; 3.4. Internet specific RRs + +;; 3.4.1. A RDATA format + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | ADDRESS | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; where: + +;; ADDRESS A 32 bit Internet address. + +;; Hosts that have multiple Internet addresses will have multiple A +;; records. + + +;; A records cause no additional section processing. The RDATA section of +;; an A line in a master file is an Internet address expressed as four +;; decimal numbers separated by dots without any imbedded spaces (e.g., +;; "10.2.0.52" or "192.0.5.6"). + +;; 3.4.2. WKS RDATA format + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | ADDRESS | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | PROTOCOL | | +;; +--+--+--+--+--+--+--+--+ | +;; | | +;; / / +;; / / +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; where: + +;; ADDRESS An 32 bit Internet address + +;; PROTOCOL An 8 bit IP protocol number + +;; A variable length bit map. The bit map must be a +;; multiple of 8 bits long. + +;; The WKS record is used to describe the well known services supported by +;; a particular protocol on a particular internet address. The PROTOCOL +;; field specifies an IP protocol number, and the bit map has one bit per +;; port of the specified protocol. The first bit corresponds to port 0, +;; the second to port 1, etc. If the bit map does not include a bit for a +;; protocol of interest, that bit is assumed zero. The appropriate values +;; and mnemonics for ports and protocols are specified in [RFC-1010]. + +;; For example, if PROTOCOL=TCP (6), the 26th bit corresponds to TCP port +;; 25 (SMTP). If this bit is set, a SMTP server should be listening on TCP +;; port 25; if zero, SMTP service is not supported on the specified +;; address. + +;; The purpose of WKS RRs is to provide availability information for +;; servers for TCP and UDP. If a server supports both TCP and UDP, or has +;; multiple Internet addresses, then multiple WKS RRs are used. + +;; WKS RRs cause no additional section processing. + +;; In master files, both ports and protocols are expressed using mnemonics +;; or decimal numbers. + +;; 3.5. IN-ADDR.ARPA domain + +;; The Internet uses a special domain to support gateway location and +;; Internet address to host mapping. Other classes may employ a similar +;; strategy in other domains. The intent of this domain is to provide a +;; guaranteed method to perform host address to host name mapping, and to +;; facilitate queries to locate all gateways on a particular network in the +;; Internet. + +;; Note that both of these services are similar to functions that could be +;; performed by inverse queries; the difference is that this part of the +;; domain name space is structured according to address, and hence can +;; guarantee that the appropriate data can be located without an exhaustive +;; search of the domain space. + +;; The domain begins at IN-ADDR.ARPA and has a substructure which follows +;; the Internet addressing structure. + +;; Domain names in the IN-ADDR.ARPA domain are defined to have up to four +;; labels in addition to the IN-ADDR.ARPA suffix. Each label represents +;; one octet of an Internet address, and is expressed as a character string +;; for a decimal value in the range 0-255 (with leading zeros omitted +;; except in the case of a zero octet which is represented by a single +;; zero). + +;; Host addresses are represented by domain names that have all four labels +;; specified. Thus data for Internet address 10.2.0.52 is located at +;; domain name 52.0.2.10.IN-ADDR.ARPA. The reversal, though awkward to +;; read, allows zones to be delegated which are exactly one network of +;; address space. For example, 10.IN-ADDR.ARPA can be a zone containing +;; data for the ARPANET, while 26.IN-ADDR.ARPA can be a separate zone for +;; MILNET. Address nodes are used to hold pointers to primary host names +;; in the normal domain space. + +;; Network numbers correspond to some non-terminal nodes at various depths +;; in the IN-ADDR.ARPA domain, since Internet network numbers are either 1, +;; 2, or 3 octets. Network nodes are used to hold pointers to the primary +;; host names of gateways attached to that network. Since a gateway is, by +;; definition, on more than one network, it will typically have two or more +;; network nodes which point at it. Gateways will also have host level +;; pointers at their fully qualified addresses. + +;; Both the gateway pointers at network nodes and the normal host pointers +;; at full address nodes use the PTR RR to point back to the primary domain +;; names of the corresponding hosts. + +;; For example, the IN-ADDR.ARPA domain will contain information about the +;; ISI gateway between net 10 and 26, an MIT gateway from net 10 to MIT's + +;; net 18, and hosts A.ISI.EDU and MULTICS.MIT.EDU. Assuming that ISI +;; gateway has addresses 10.2.0.22 and 26.0.0.103, and a name MILNET- +;; GW.ISI.EDU, and the MIT gateway has addresses 10.0.0.77 and 18.10.0.4 +;; and a name GW.LCS.MIT.EDU, the domain database would contain: + +;; 10.IN-ADDR.ARPA. PTR MILNET-GW.ISI.EDU. +;; 10.IN-ADDR.ARPA. PTR GW.LCS.MIT.EDU. +;; 18.IN-ADDR.ARPA. PTR GW.LCS.MIT.EDU. +;; 26.IN-ADDR.ARPA. PTR MILNET-GW.ISI.EDU. +;; 22.0.2.10.IN-ADDR.ARPA. PTR MILNET-GW.ISI.EDU. +;; 103.0.0.26.IN-ADDR.ARPA. PTR MILNET-GW.ISI.EDU. +;; 77.0.0.10.IN-ADDR.ARPA. PTR GW.LCS.MIT.EDU. +;; 4.0.10.18.IN-ADDR.ARPA. PTR GW.LCS.MIT.EDU. +;; 103.0.3.26.IN-ADDR.ARPA. PTR A.ISI.EDU. +;; 6.0.0.10.IN-ADDR.ARPA. PTR MULTICS.MIT.EDU. + +;; Thus a program which wanted to locate gateways on net 10 would originate +;; a query of the form QTYPE=PTR, QCLASS=IN, QNAME=10.IN-ADDR.ARPA. It +;; would receive two RRs in response: + +;; 10.IN-ADDR.ARPA. PTR MILNET-GW.ISI.EDU. +;; 10.IN-ADDR.ARPA. PTR GW.LCS.MIT.EDU. + +;; The program could then originate QTYPE=A, QCLASS=IN queries for MILNET- +;; GW.ISI.EDU. and GW.LCS.MIT.EDU. to discover the Internet addresses of +;; these gateways. + +;; A resolver which wanted to find the host name corresponding to Internet +;; host address 10.0.0.6 would pursue a query of the form QTYPE=PTR, +;; QCLASS=IN, QNAME=6.0.0.10.IN-ADDR.ARPA, and would receive: + +;; 6.0.0.10.IN-ADDR.ARPA. PTR MULTICS.MIT.EDU. + +;; Several cautions apply to the use of these services: +;; - Since the IN-ADDR.ARPA special domain and the normal domain +;; for a particular host or gateway will be in different zones, +;; the possibility exists that that the data may be inconsistent. + +;; - Gateways will often have two names in separate domains, only +;; one of which can be primary. + +;; - Systems that use the domain database to initialize their +;; routing tables must start with enough gateway information to +;; guarantee that they can access the appropriate name server. + +;; - The gateway data only reflects the existence of a gateway in a +;; manner equivalent to the current HOSTS.TXT file. It doesn't +;; replace the dynamic availability information from GGP or EGP. + +;; 3.6. Defining new types, classes, and special namespaces + +;; The previously defined types and classes are the ones in use as of the +;; date of this memo. New definitions should be expected. This section +;; makes some recommendations to designers considering additions to the +;; existing facilities. The mailing list NAMEDROPPERS@SRI-NIC.ARPA is the +;; forum where general discussion of design issues takes place. + +;; In general, a new type is appropriate when new information is to be +;; added to the database about an existing object, or we need new data +;; formats for some totally new object. Designers should attempt to define +;; types and their RDATA formats that are generally applicable to all +;; classes, and which avoid duplication of information. New classes are +;; appropriate when the DNS is to be used for a new protocol, etc which +;; requires new class-specific data formats, or when a copy of the existing +;; name space is desired, but a separate management domain is necessary. + +;; New types and classes need mnemonics for master files; the format of the +;; master files requires that the mnemonics for type and class be disjoint. + +;; TYPE and CLASS values must be a proper subset of QTYPEs and QCLASSes +;; respectively. + +;; The present system uses multiple RRs to represent multiple values of a +;; type rather than storing multiple values in the RDATA section of a +;; single RR. This is less efficient for most applications, but does keep +;; RRs shorter. The multiple RRs assumption is incorporated in some +;; experimental work on dynamic update methods. + +;; The present system attempts to minimize the duplication of data in the +;; database in order to insure consistency. Thus, in order to find the +;; address of the host for a mail exchange, you map the mail domain name to +;; a host name, then the host name to addresses, rather than a direct +;; mapping to host address. This approach is preferred because it avoids +;; the opportunity for inconsistency. + +;; In defining a new type of data, multiple RR types should not be used to +;; create an ordering between entries or express different formats for +;; equivalent bindings, instead this information should be carried in the +;; body of the RR and a single type used. This policy avoids problems with +;; caching multiple types and defining QTYPEs to match multiple types. + +;; For example, the original form of mail exchange binding used two RR +;; types one to represent a "closer" exchange (MD) and one to represent a +;; "less close" exchange (MF). The difficulty is that the presence of one +;; RR type in a cache doesn't convey any information about the other +;; because the query which acquired the cached information might have used +;; a QTYPE of MF, MD, or MAILA (which matched both). The redesigned + + + + +;; service used a single type (MX) with a "preference" value in the RDATA +;; section which can order different RRs. However, if any MX RRs are found +;; in the cache, then all should be there. + +;; 4. MESSAGES + +;; 4.1. Format + +;; All communications inside of the domain protocol are carried in a single +;; format called a message. The top level format of message is divided +;; into 5 sections (some of which are empty in certain cases) shown below: + +;; The names of the sections after the header are derived from their use in +;; standard queries. The question section contains fields that describe a +;; question to a name server. These fields are a query type (QTYPE), a +;; query class (QCLASS), and a query domain name (QNAME). The last three +;; sections have the same format: a possibly empty list of concatenated +;; resource records (RRs). The answer section contains RRs that answer the +;; question; the authority section contains RRs that point toward an +;; authoritative name server; the additional records section contains RRs +;; which relate to the query, but are not strictly answers for the +;; question. + +;; The header section is always present. The header includes fields that +;; specify which of the remaining sections are present, and also specify +;; whether the message is a query or a response, a standard query or some +;; other opcode, etc. + +;; 4.1.1. Header section format + +(defbinary dns-header (:byte-order :big-endian) + ;; A 16 bit identifier assigned by the program that + ;; generates any kind of query. This identifier is copied + ;; the corresponding reply and can be used by the requester + ;; to match up replies to outstanding queries. + (id 0 :type 16) + + ;; A one bit field that specifies whether this message is a + ;; query (0), or a response (1). + (qr 0 :type 1) + + ;; A four bit field that specifies kind of query in this + ;; message. This value is set by the originator of a query + ;; and copied into the response. The values are: + ;; + ;; 0 a standard query (QUERY) + ;; 1 an inverse query (IQUERY) + ;; 2 a server status request (STATUS) + ;; 3-15 reserved for future use + (opcode 0 :type 4) ; TODO(tazjin): use define-enum + + ;; Authoritative Answer - this bit is valid in responses, + ;; and specifies that the responding name server is an + ;; authority for the domain name in question section. + (aa nil :type 1) + + ;; TrunCation - specifies that this message was truncated + ;; due to length greater than that permitted on the + ;; transmission channel. + (tc nil :type 1) + + ;; Recursion Desired - this bit may be set in a query and + ;; is copied into the response. If RD is set, it directs + ;; the name server to pursue the query recursively. + ;; Recursive query support is optional. + (rd nil :type 1) + + ;; Recursion Available - this be is set or cleared in a + ;; response, and denotes whether recursive query support is + ;; available in the name server. + (ra nil :type 1) + + ;; Reserved for future use. Must be zero in all queries and + ;; responses. + (z 0 :type 3) + + ;; Response code - this 4 bit field is set as part of + ;; responses. The values have the following + ;; interpretation: + ;; 0 No error condition + ;; 1 Format error - The name server was + ;; unable to interpret the query. + ;; 2 Server failure - The name server was + ;; unable to process this query due to a + ;; problem with the name server. + ;; 3 Name Error - Meaningful only for + ;; responses from an authoritative name + ;; server, this code signifies that the + ;; domain name referenced in the query does + ;; not exist. + ;; 4 Not Implemented - The name server does + ;; not support the requested kind of query. + ;; 5 Refused - The name server refuses to + ;; perform the specified operation for + ;; policy reasons. For example, a name + ;; server may not wish to provide the + ;; information to the particular requester, + ;; or a name server may not wish to perform + ;; a particular operation (e.g., zone + ;; transfer) for particular data. + ;; 6-15 Reserved for future use. + (rcode 0 :type 4) + + ;; an unsigned 16 bit integer specifying the number of + ;; entries in the question section. + (qdcount 0 :type 16) + + ;; an unsigned 16 bit integer specifying the number of + ;; resource records in the answer section. + (ancount 0 :type 16) + + ;; an unsigned 16 bit integer specifying the number of name + ;; server resource records in the authority records + ;; section. + (nscount 0 :type 16) + + ;; an unsigned 16 bit integer specifying the number of + ;; resource records in the additional records section. + (arcount 0 :type 16)) + +;; Domain names in questions and resource records are represented as a +;; sequence of labels, where each label consists of a length octet +;; followed by that number of octets. +;; +;; The domain name terminates with the zero length octet for the null +;; label of the root. Note that this field may be an odd number of +;; octets; no padding is used. +(declaim (ftype (function (stream) (values (vector string) integer)) read-qname)) +(defun read-qname (stream) + "Reads a DNS QNAME from STREAM." + (format t "reading qname at ~A" (file-position stream)) + (iter (for byte in-stream stream using #'read-byte) + ;; Total size is needed, count for each iteration byte, plus its + ;; own value. + (sum (+ 1 byte) into size) + + (until (equal byte 0)) + + ;; Each fragment is collected into this byte vector pre-allocated + ;; with the correct size. + (for fragment = (make-array byte :element-type '(unsigned-byte 8) + :fill-pointer 0)) + + ;; On each iteration, this will interpret the current byte as an + ;; unsigned integer and read from STREAM an equivalent amount of + ;; times to assemble the current fragment. + ;; + ;; Advancing the stream like this also ensures that the next + ;; iteration occurs on either a length-byte or the final + ;; terminating byte. + (dotimes (_ byte (collect (babel:octets-to-string fragment) + into fragments result-type vector)) + (vector-push (read-byte stream) fragment)) + + (finally (return (values fragments size))))) + +(declaim (ftype (function (stream (vector string))) write-qname)) +(defun write-qname (stream qname) + "Write a DNS qname to STREAM." + + ;; Write each fragment starting with its (byte-) length, followed by + ;; the bytes. + (iter (for fragment in-vector qname) + (for bytes = (babel:string-to-octets fragment)) + (write-byte (length bytes) stream) + (iter (for byte in-vector bytes) + (write-byte byte stream))) + + ;; Always finish off the serialisation with a null-byte! + (write-byte 0 stream)) + +;; 4.1.2. Question section format +(defbinary dns-question (:byte-order :big-endian) + ;; a domain name represented + (qname "" :type (custom :lisp-type (vector string) + :reader #'read-qname + :writer #'write-qname)) + + ;; a two octet code which specifies the type of the query. + ;; The values for this field include all codes valid for a + ;; TYPE field, together with some more general codes which + ;; can match more than one type of RR. + (qtype 0 :type 16) ;; TODO(tazjin): define type after the RR binary + + ;; a two octet code that specifies the class of the query. For + ;; example, the QCLASS field is IN for the Internet. + (qclass 0 :type 16)) ; TODO(tazjin): enum? + +;; 4.1.3. Resource record format + +(defbinary dns-rr (:byte-order :big-endian) + ;; magic number indicating a pointer response + ;; + ;; TODO(tazjin): This could theoretically be a QNAME, but + ;; Google DNS doesn't do that. For compatibility it is + ;; still sensible to add support for it. + (magic 3 :type (magic :value 3 :actual-type (unsigned-byte 2))) + + ;; a domain name to which this resource record pertains. + (name nil :type (pointer :data-type (custom :lisp-type (vector string) + :reader #'read-qname + :writer #'write-qname) + :pointer-type (unsigned-byte 14))) + + ;; two octets containing one of the RR type codes. This + ;; field specifies the meaning of the data in the RDATA + ;; field. + (type 0 :type 16) ; TODO(tazjin): enum? + + ;; two octets which specify the class of the data in the + ;; RDATA field. + (class 0 :type 16) ; TODO(tazjin): enum + + ;; a 32 bit unsigned integer that specifies the time + ;; interval (in seconds) that the resource record may be + ;; cached before it should be discarded. Zero values are + ;; interpreted to mean that the RR can only be used for the + ;; transaction in progress, and should not be cached. + (ttl 0 :type 32) + + ;; an unsigned 16 bit integer that specifies the length in + ;; octets of the RDATA field. + (rdlength 0 :type 16) + + ;; a variable length string of octets that describes the + ;; resource. The format of this information varies + ;; according to the TYPE and CLASS of the resource record. + ;; For example, the if the TYPE is A and the CLASS is IN, + ;; the RDATA field is a 4 octet ARPA Internet address. + (rdata #() :type (simple-array (unsigned-byte 8) (rdlength)))) + +(defbinary dns-message (:byte-order :big-endian) + (header nil :type dns-header) + + ;; the question for the name server + (question #() :type (simple-array dns-question ((dns-header-qdcount header)))) + + ;; RRs answering the question + (answer #() :type (simple-array dns-rr ((dns-header-ancount header)))) + + ;; ;; RRs pointing toward an authority + ;; (authority) + + ;; ;; RRs holding additional information + ;; (additional) + ) + +;; 4.1.4. Message compression + +;; In order to reduce the size of messages, the domain system utilizes a +;; compression scheme which eliminates the repetition of domain names in a +;; message. In this scheme, an entire domain name or a list of labels at +;; the end of a domain name is replaced with a pointer to a prior occurance +;; of the same name. + +;; The pointer takes the form of a two octet sequence: + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | 1 1| OFFSET | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; The first two bits are ones. This allows a pointer to be distinguished +;; from a label, since the label must begin with two zero bits because +;; labels are restricted to 63 octets or less. (The 10 and 01 combinations +;; are reserved for future use.) The OFFSET field specifies an offset from +;; the start of the message (i.e., the first octet of the ID field in the +;; domain header). A zero offset specifies the first byte of the ID field, +;; etc. + +;; The compression scheme allows a domain name in a message to be +;; represented as either: + +;; - a sequence of labels ending in a zero octet + +;; - a pointer + +;; - a sequence of labels ending with a pointer + +;; Pointers can only be used for occurances of a domain name where the +;; format is not class specific. If this were not the case, a name server +;; or resolver would be required to know the format of all RRs it handled. +;; As yet, there are no such cases, but they may occur in future RDATA +;; formats. + +;; If a domain name is contained in a part of the message subject to a +;; length field (such as the RDATA section of an RR), and compression is +;; used, the length of the compressed name is used in the length +;; calculation, rather than the length of the expanded name. + +;; Programs are free to avoid using pointers in messages they generate, +;; although this will reduce datagram capacity, and may cause truncation. +;; However all programs are required to understand arriving messages that +;; contain pointers. + +;; For example, a datagram might need to use the domain names F.ISI.ARPA, +;; FOO.F.ISI.ARPA, ARPA, and the root. Ignoring the other fields of the +;; message, these domain names might be represented as: + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; 20 | 1 | F | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; 22 | 3 | I | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; 24 | S | I | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; 26 | 4 | A | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; 28 | R | P | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; 30 | A | 0 | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; 40 | 3 | F | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; 42 | O | O | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; 44 | 1 1| 20 | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; 64 | 1 1| 26 | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; 92 | 0 | | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; The domain name for F.ISI.ARPA is shown at offset 20. The domain name +;; FOO.F.ISI.ARPA is shown at offset 40; this definition uses a pointer to +;; concatenate a label for FOO to the previously defined F.ISI.ARPA. The +;; domain name ARPA is defined at offset 64 using a pointer to the ARPA +;; component of the name F.ISI.ARPA at 20; note that this pointer relies on +;; ARPA being the last label in the string at 20. The root domain name is +;; defined by a single octet of zeros at 92; the root domain name has no +;; labels. + +;; 4.2. Transport +;; Messages sent over TCP connections use server port 53 (decimal). The +;; message is prefixed with a two byte length field which gives the message +;; length, excluding the two byte length field. This length field allows +;; the low-level processing to assemble a complete message before beginning +;; to parse it. From cefb60f20cf0fe8d1540402848b2c93fa976e9f1 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 26 Jan 2020 01:36:40 +0000 Subject: [PATCH 3/8] refactor(lisp/dns): Introduce structured QNAME representation Adds a struct that represents QNAMEs, tracks the stream offset at which the QNAME parsing began and makes it possible to resolve pointers inside of the QNAME. Note that resolving pointers needs to happen *after* the call to lisp-binary currently. It might be possible to implement this inside of lisp-binary in the future by switching on the top two bits of the qname field, but since this is happening *inside* of a reader function I'm not currently sure how to implement it. --- lisp/dns/message.lisp | 72 +++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/lisp/dns/message.lisp b/lisp/dns/message.lisp index 9e283ea5c..079e971c7 100644 --- a/lisp/dns/message.lisp +++ b/lisp/dns/message.lisp @@ -706,6 +706,24 @@ ;; resource records in the additional records section. (arcount 0 :type 16)) + +;; Representation of DNS QNAMEs. +;; +;; A QNAME can be either made up entirely of labels, which is +;; basically a list of strings, or be terminated with a pointer to an +;; offset within the original message. + +(deftype qname-field () + '(or + ;; pointer + (unsigned-byte 14) + ;; label + string)) + +(defstruct qname + (start-at 0 :type (unsigned-byte 14)) + (names #() :type (vector qname-field))) + ;; Domain names in questions and resource records are represented as a ;; sequence of labels, where each label consists of a length octet ;; followed by that number of octets. @@ -713,42 +731,44 @@ ;; The domain name terminates with the zero length octet for the null ;; label of the root. Note that this field may be an odd number of ;; octets; no padding is used. -(declaim (ftype (function (stream) (values (vector string) integer)) read-qname)) +(declaim (ftype (function (stream) (values qname integer)) read-qname)) (defun read-qname (stream) "Reads a DNS QNAME from STREAM." - (format t "reading qname at ~A" (file-position stream)) - (iter (for byte in-stream stream using #'read-byte) - ;; Total size is needed, count for each iteration byte, plus its - ;; own value. - (sum (+ 1 byte) into size) + (let ((start-at (+ 1 (file-position stream)))) + (iter (for byte in-stream stream using #'read-byte) + ;; Total size is needed, count for each iteration byte, plus its + ;; own value. + (sum (+ 1 byte) into size) - (until (equal byte 0)) + (until (equal byte 0)) - ;; Each fragment is collected into this byte vector pre-allocated - ;; with the correct size. - (for fragment = (make-array byte :element-type '(unsigned-byte 8) - :fill-pointer 0)) + ;; Each fragment is collected into this byte vector pre-allocated + ;; with the correct size. + (for fragment = (make-array byte :element-type '(unsigned-byte 8) + :fill-pointer 0)) - ;; On each iteration, this will interpret the current byte as an - ;; unsigned integer and read from STREAM an equivalent amount of - ;; times to assemble the current fragment. - ;; - ;; Advancing the stream like this also ensures that the next - ;; iteration occurs on either a length-byte or the final - ;; terminating byte. - (dotimes (_ byte (collect (babel:octets-to-string fragment) - into fragments result-type vector)) - (vector-push (read-byte stream) fragment)) + ;; On each iteration, this will interpret the current byte as an + ;; unsigned integer and read from STREAM an equivalent amount of + ;; times to assemble the current fragment. + ;; + ;; Advancing the stream like this also ensures that the next + ;; iteration occurs on either a length-byte or the final + ;; terminating byte. + (dotimes (_ byte (collect (babel:octets-to-string fragment) + into fragments result-type vector)) + (vector-push (read-byte stream) fragment)) - (finally (return (values fragments size))))) + (finally (return (values (make-qname :start-at start-at + :names fragments) + size)))))) -(declaim (ftype (function (stream (vector string))) write-qname)) +(declaim (ftype (function (stream qname)) write-qname)) (defun write-qname (stream qname) "Write a DNS qname to STREAM." ;; Write each fragment starting with its (byte-) length, followed by ;; the bytes. - (iter (for fragment in-vector qname) + (iter (for fragment in-vector (qname-names qname)) (for bytes = (babel:string-to-octets fragment)) (write-byte (length bytes) stream) (iter (for byte in-vector bytes) @@ -760,7 +780,7 @@ ;; 4.1.2. Question section format (defbinary dns-question (:byte-order :big-endian) ;; a domain name represented - (qname "" :type (custom :lisp-type (vector string) + (qname "" :type (custom :lisp-type qname :reader #'read-qname :writer #'write-qname)) @@ -785,7 +805,7 @@ (magic 3 :type (magic :value 3 :actual-type (unsigned-byte 2))) ;; a domain name to which this resource record pertains. - (name nil :type (pointer :data-type (custom :lisp-type (vector string) + (name nil :type (pointer :data-type (custom :lisp-type qname :reader #'read-qname :writer #'write-qname) :pointer-type (unsigned-byte 14))) From 1440fc0dd722dded073888c9bc3bd5101774309d Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 26 Jan 2020 18:27:35 +0000 Subject: [PATCH 4/8] feat(lisp/dns): Implement qname compression parsing Implements support for the compresion scheme used in binary DNS messages. This makes it possible to decode messages entirely, but not yet actually resolve the labels to their "real" values. All qnames are stored with file-offsets pointing at the position at which their reading started, which enables the implementation of a function to resolve pointers internally. --- lisp/dns/message.lisp | 299 ++++-------------------------------------- 1 file changed, 29 insertions(+), 270 deletions(-) diff --git a/lisp/dns/message.lisp b/lisp/dns/message.lisp index 079e971c7..0eedbe660 100644 --- a/lisp/dns/message.lisp +++ b/lisp/dns/message.lisp @@ -1,152 +1,5 @@ (in-package :dns) -;; 3. DOMAIN NAME SPACE AND RR DEFINITIONS 10 -;; 3.1. Name space definitions 10 -;; 3.2. RR definitions 11 -;; 3.2.1. Format 11 -;; 3.2.2. TYPE values 12 -;; 3.2.3. QTYPE values 12 -;; 3.2.4. CLASS values 13 -;; 3.2.5. QCLASS values 13 -;; 3.3. Standard RRs 13 -;; 3.3.1. CNAME RDATA format 14 -;; 3.3.2. HINFO RDATA format 14 -;; 3.3.3. MB RDATA format (EXPERIMENTAL) 14 -;; 3.3.4. MD RDATA format (Obsolete) 15 -;; 3.3.5. MF RDATA format (Obsolete) 15 -;; 3.3.6. MG RDATA format (EXPERIMENTAL) 16 -;; 3.3.7. MINFO RDATA format (EXPERIMENTAL) 16 -;; 3.3.8. MR RDATA format (EXPERIMENTAL) 17 -;; 3.3.9. MX RDATA format 17 -;; 3.3.10. NULL RDATA format (EXPERIMENTAL) 17 -;; 3.3.11. NS RDATA format 18 -;; 3.3.12. PTR RDATA format 18 -;; 3.3.13. SOA RDATA format 19 -;; 3.3.14. TXT RDATA format 20 -;; 3.4. ARPA Internet specific RRs 20 -;; 3.4.1. A RDATA format 20 -;; 3.4.2. WKS RDATA format 21 -;; 3.5. IN-ADDR.ARPA domain 22 -;; 3.6. Defining new types, classes, and special namespaces 24 -;; 4. MESSAGES 25 -;; 4.1. Format 25 -;; 4.1.1. Header section format 26 -;; 4.1.2. Question section format 28 -;; 4.1.3. Resource record format 29 -;; 4.1.4. Message compression 30 -;; 4.2. Transport 32 -;; 4.2.1. UDP usage 32 -;; 4.2.2. TCP usage 32 -;; 5. MASTER FILES 33 -;; 5.1. Format 33 -;; 5.2. Use of master files to define zones 35 -;; 5.3. Master file example 36 -;; 6. NAME SERVER IMPLEMENTATION 37 -;; 6.1. Architecture 37 -;; 6.1.1. Control 37 -;; 6.1.2. Database 37 -;; 6.1.3. Time 39 -;; 6.2. Standard query processing 39 -;; 6.3. Zone refresh and reload processing 39 -;; 6.4. Inverse queries (Optional) 40 -;; 6.4.1. The contents of inverse queries and responses 40 -;; 6.4.2. Inverse query and response example 41 -;; 6.4.3. Inverse query processing 42 -;; 6.5. Completion queries and responses 42 -;; 7. RESOLVER IMPLEMENTATION 43 -;; 7.1. Transforming a user request into a query 43 -;; 7.2. Sending the queries 44 -;; 7.3. Processing responses 46 -;; 7.4. Using the cache 47 -;; 8. MAIL SUPPORT 47 -;; 8.1. Mail exchange binding 48 -;; 8.2. Mailbox binding (Experimental) 48 -;; 9. REFERENCES and BIBLIOGRAPHY 50 -;; Index 54 - -;; 2.3.4. Size limits -;; Various objects and parameters in the DNS have size limits. They are -;; listed below. Some could be easily changed, others are more -;; fundamental. -;; labels 63 octets or less -;; names 255 octets or less -;; TTL positive values of a signed 32 bit number. -;; UDP messages 512 octets or less - -;; 3. DOMAIN NAME SPACE AND RR DEFINITIONS - -;; Domain names in messages are expressed in terms of a sequence of labels. -;; Each label is represented as a one octet length field followed by that -;; number of octets. Since every domain name ends with the null label of -;; the root, a domain name is terminated by a length byte of zero. The -;; high order two bits of every length octet must be zero, and the -;; remaining six bits of the length field limit the label to 63 octets or -;; less. - -;; To simplify implementations, the total length of a domain name (i.e., -;; label octets and label length octets) is restricted to 255 octets or -;; less. - -;; Although labels can contain any 8 bit values in octets that make up a -;; label, it is strongly recommended that labels follow the preferred -;; syntax described elsewhere in this memo, which is compatible with -;; existing host naming conventions. Name servers and resolvers must -;; compare labels in a case-insensitive manner (i.e., A=a), assuming ASCII -;; with zero parity. Non-alphabetic codes must match exactly. - -;; 3.2. RR definitions - -;; 3.2.1. Format - -;; All RRs have the same top level format shown below: - -;; 1 1 1 1 1 1 -;; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; | | -;; / / -;; / NAME / -;; | | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; | TYPE | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; | CLASS | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; | TTL | -;; | | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; | RDLENGTH | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--| -;; / RDATA / -;; / / -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ - - -;; where: - -;; NAME an owner name, i.e., the name of the node to which this -;; resource record pertains. - -;; TYPE two octets containing one of the RR TYPE codes. - -;; CLASS two octets containing one of the RR CLASS codes. - -;; TTL a 32 bit signed integer that specifies the time interval -;; that the resource record may be cached before the source -;; of the information should again be consulted. Zero -;; values are interpreted to mean that the RR can only be -;; used for the transaction in progress, and should not be -;; cached. For example, SOA records are always distributed -;; with a zero TTL to prohibit caching. Zero values can -;; also be used for extremely volatile data. - -;; RDLENGTH an unsigned 16 bit integer that specifies the length in -;; octets of the RDATA field. - -;; RDATA a variable length string of octets that describes the -;; resource. The format of this information varies -;; according to the TYPE and CLASS of the resource record. - ;; 3.2.2. TYPE values ;; TYPE fields are used in resource records. Note that these types are a @@ -734,26 +587,38 @@ (declaim (ftype (function (stream) (values qname integer)) read-qname)) (defun read-qname (stream) "Reads a DNS QNAME from STREAM." - (let ((start-at (+ 1 (file-position stream)))) - (iter (for byte in-stream stream using #'read-byte) - ;; Total size is needed, count for each iteration byte, plus its - ;; own value. - (sum (+ 1 byte) into size) - - (until (equal byte 0)) + (let ((start-at (file-position stream))) + (iter (for byte next (read-byte stream)) ;; Each fragment is collected into this byte vector pre-allocated ;; with the correct size. (for fragment = (make-array byte :element-type '(unsigned-byte 8) :fill-pointer 0)) + ;; If the bit sequence (1 1) is encountered at the beginning of + ;; the fragment, a qname pointer is being read. + (let ((byte-copy byte)) + (when (equal #b11 (lisp-binary/integer:pop-bits 2 8 byte-copy)) + (let ((next (read-byte stream))) + (lisp-binary/integer:push-bits byte-copy 8 next) + (collect next into fragments result-type vector) + (sum 2 into size) + (finish)))) + + ;; Total size is needed, count for each iteration byte, plus its + ;; own value. + (sum (+ 1 byte) into size) + (until (equal byte 0)) + ;; On each iteration, this will interpret the current byte as an ;; unsigned integer and read from STREAM an equivalent amount of ;; times to assemble the current fragment. ;; ;; Advancing the stream like this also ensures that the next - ;; iteration occurs on either a length-byte or the final - ;; terminating byte. + ;; iteration occurs on a new fragment or the final terminating + ;; byte. + ;; + ;; TODO(tazjin): Use lisp-binary:read-counted-string. (dotimes (_ byte (collect (babel:octets-to-string fragment) into fragments result-type vector)) (vector-push (read-byte stream) fragment)) @@ -797,18 +662,9 @@ ;; 4.1.3. Resource record format (defbinary dns-rr (:byte-order :big-endian) - ;; magic number indicating a pointer response - ;; - ;; TODO(tazjin): This could theoretically be a QNAME, but - ;; Google DNS doesn't do that. For compatibility it is - ;; still sensible to add support for it. - (magic 3 :type (magic :value 3 :actual-type (unsigned-byte 2))) - - ;; a domain name to which this resource record pertains. - (name nil :type (pointer :data-type (custom :lisp-type qname - :reader #'read-qname - :writer #'write-qname) - :pointer-type (unsigned-byte 14))) + (name nil :type (custom :lisp-type qname + :reader #'read-qname + :writer #'write-qname)) ;; two octets containing one of the RR type codes. This ;; field specifies the meaning of the data in the RDATA @@ -843,109 +699,12 @@ ;; the question for the name server (question #() :type (simple-array dns-question ((dns-header-qdcount header)))) - ;; RRs answering the question + ;; ;; RRs answering the question + ;; (answer #() :type (simple-array (unsigned-byte 8) (16))) (answer #() :type (simple-array dns-rr ((dns-header-ancount header)))) - ;; ;; RRs pointing toward an authority - ;; (authority) + ;; ;; ;; RRs pointing toward an authority + (authority #() :type (simple-array dns-rr ((dns-header-nscount header)))) ;; ;; RRs holding additional information - ;; (additional) - ) - -;; 4.1.4. Message compression - -;; In order to reduce the size of messages, the domain system utilizes a -;; compression scheme which eliminates the repetition of domain names in a -;; message. In this scheme, an entire domain name or a list of labels at -;; the end of a domain name is replaced with a pointer to a prior occurance -;; of the same name. - -;; The pointer takes the form of a two octet sequence: - -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; | 1 1| OFFSET | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ - -;; The first two bits are ones. This allows a pointer to be distinguished -;; from a label, since the label must begin with two zero bits because -;; labels are restricted to 63 octets or less. (The 10 and 01 combinations -;; are reserved for future use.) The OFFSET field specifies an offset from -;; the start of the message (i.e., the first octet of the ID field in the -;; domain header). A zero offset specifies the first byte of the ID field, -;; etc. - -;; The compression scheme allows a domain name in a message to be -;; represented as either: - -;; - a sequence of labels ending in a zero octet - -;; - a pointer - -;; - a sequence of labels ending with a pointer - -;; Pointers can only be used for occurances of a domain name where the -;; format is not class specific. If this were not the case, a name server -;; or resolver would be required to know the format of all RRs it handled. -;; As yet, there are no such cases, but they may occur in future RDATA -;; formats. - -;; If a domain name is contained in a part of the message subject to a -;; length field (such as the RDATA section of an RR), and compression is -;; used, the length of the compressed name is used in the length -;; calculation, rather than the length of the expanded name. - -;; Programs are free to avoid using pointers in messages they generate, -;; although this will reduce datagram capacity, and may cause truncation. -;; However all programs are required to understand arriving messages that -;; contain pointers. - -;; For example, a datagram might need to use the domain names F.ISI.ARPA, -;; FOO.F.ISI.ARPA, ARPA, and the root. Ignoring the other fields of the -;; message, these domain names might be represented as: - -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; 20 | 1 | F | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; 22 | 3 | I | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; 24 | S | I | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; 26 | 4 | A | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; 28 | R | P | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; 30 | A | 0 | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ - -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; 40 | 3 | F | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; 42 | O | O | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; 44 | 1 1| 20 | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ - -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; 64 | 1 1| 26 | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ - -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ -;; 92 | 0 | | -;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ - -;; The domain name for F.ISI.ARPA is shown at offset 20. The domain name -;; FOO.F.ISI.ARPA is shown at offset 40; this definition uses a pointer to -;; concatenate a label for FOO to the previously defined F.ISI.ARPA. The -;; domain name ARPA is defined at offset 64 using a pointer to the ARPA -;; component of the name F.ISI.ARPA at 20; note that this pointer relies on -;; ARPA being the last label in the string at 20. The root domain name is -;; defined by a single octet of zeros at 92; the root domain name has no -;; labels. - -;; 4.2. Transport -;; Messages sent over TCP connections use server port 53 (decimal). The -;; message is prefixed with a two byte length field which gives the message -;; length, excluding the two byte length field. This length field allows -;; the low-level processing to assemble a complete message before beginning -;; to parse it. + (additional #() :type (simple-array dns-rr ((dns-header-arcount header))))) From 8f805a29d10ca7080aec0f07feca49fc22a631a3 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 26 Jan 2020 19:58:52 +0000 Subject: [PATCH 5/8] feat(lisp/dns): Use new DNS deserialiser in dns:lookup-generic This enables arbitrary DNS lookups (with the caveat that RRDATAs are currently not deserialised into a record-type-specific format). An error condition has been defined for error-responses from the HTTP server which provides interactive restarts for attempting a new call with different parameters. --- lisp/dns/client.lisp | 204 +++++++++++-------------------------------- 1 file changed, 52 insertions(+), 152 deletions(-) diff --git a/lisp/dns/client.lisp b/lisp/dns/client.lisp index 0f355589d..2dbe9ff31 100644 --- a/lisp/dns/client.lisp +++ b/lisp/dns/client.lisp @@ -4,19 +4,65 @@ (in-package #:dns) ;; The DoH client is configured with a URI Template [RFC6570] -(defvar *doh-base-url* "https://dns.google/dns-query" +(defvar *doh-base-url* "https://dns.google/resolve" "Base URL of the service providing DNS-over-HTTP(S). Defaults to the Google-hosted API.") -(defun lookup-generic (name type) - (multiple-value-bind (stream) - (drakma:http-request *doh-base-url* +(define-condition doh-error (error) + ((query-name :initarg :query-name + :reader doh-error-query-name + :type string) + (query-type :initarg :query-type + :reader doh-error-query-type + :type string) + (doh-url :initarg :doh-url + :reader doh-error-doh-url + :type string) + (status-code :initarg :status-code + :reader doh-error-status-code + :type integer) + (response-body :initarg :response-body + :reader doh-error-response-body + :type (or nil (vector (unsigned-byte 8)) string))) + + (:report (lambda (condition stream) + (let ((url (doh-error-doh-url condition)) + (status (doh-error-status-code condition)) + (body (doh-error-response-body condition))) + (format stream "DoH service at '~A' responded with non-success (~A): ~%~%~A" + url status body))))) + +(defun lookup-generic (name type &key (doh-url *doh-base-url*)) + (multiple-value-bind (body status) + (drakma:http-request doh-url :decode-content t - :want-stream t + ;; TODO(tazjin): Figure out why 'want-stream' doesn't work :parameters `(("type" . ,type) ("name" . ,name) ("ct" . "application/dns-message"))) - (read-binary 'dns-message stream))) + (if (= 200 status) + (read-binary 'dns-message (flexi-streams:make-in-memory-input-stream body)) + + (restart-case (error 'doh-error + :query-name name + :query-type type + :doh-url doh-url + :status-code status + :response-body body) + (call-with-other-name (new-name) + :interactive (lambda () (list (the string (read)))) + :test (lambda (c) (typep c 'doh-error)) + (lookup-generic new-name type :doh-url doh-url)) + + (call-with-other-type (new-type) + :interactive (lambda () (list (the string (read)))) + :test (lambda (c) (typep c 'doh-error)) + (lookup-generic name new-type :doh-url doh-url)) + + (call-with-other-url (new-url) + :interactive (lambda () (list (the string (read)))) + :test (lambda (c) (typep c 'doh-error)) + (lookup-generic name type :doh-url new-url)))))) (defun lookup-txt (name) "Look up the TXT records at NAME." @@ -25,149 +71,3 @@ (defun lookup-mx (name) "Look up the MX records at NAME." (lookup-generic name "MX")) - - -;; The URI Template defined in this document is processed without any -;; variables when the HTTP method is POST. When the HTTP method is GET, -;; the single variable "dns" is defined as the content of the DNS -;; request (as described in Section 6), encoded with base64url -;; [RFC4648]. - -;; When using the POST method, the DNS query is included as the message -;; body of the HTTP request, and the Content-Type request header field -;; indicates the media type of the message. POSTed requests are -;; generally smaller than their GET equivalents. - -;; Using the GET method is friendlier to many HTTP cache -;; implementations. - -;; The DoH client SHOULD include an HTTP Accept request header field to -;; indicate what type of content can be understood in response. -;; Irrespective of the value of the Accept request header field, the -;; client MUST be prepared to process "application/dns-message" (as -;; described in Section 6) responses but MAY also process other DNS- -;; related media types it receives. - -;; In order to maximize HTTP cache friendliness, DoH clients using media -;; formats that include the ID field from the DNS message header, such -;; as "application/dns-message", SHOULD use a DNS ID of 0 in every DNS -;; request. HTTP correlates the request and response, thus eliminating -;; the need for the ID in a media type such as "application/dns- -;; message". The use of a varying DNS ID can cause semantically -;; equivalent DNS queries to be cached separately. - -;; DoH clients can use HTTP/2 padding and compression [RFC7540] in the -;; same way that other HTTP/2 clients use (or don't use) them. - -;; 4.1.1. HTTP Request Examples - -;; These examples use HTTP/2-style formatting from [RFC7540]. - -;; These examples use a DoH service with a URI Template of -;; "https://dnsserver.example.net/dns-query{?dns}" to resolve IN A -;; records. - -;; The requests are represented as bodies with media type "application/ -;; dns-message". - -;; The first example request uses GET to request "www.example.com". - -;; :method = GET -;; :scheme = https -;; :authority = dnsserver.example.net -;; :path = /dns-query?dns=AAABAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB -;; accept = application/dns-message - -;; Finally, a GET-based query for "a.62characterlabel-makes-base64url- -;; distinct-from-standard-base64.example.com" is shown as an example to -;; emphasize that the encoding alphabet of base64url is different than -;; regular base64 and that padding is omitted. - -;; The only response type defined in this document is "application/dns- -;; message", but it is possible that other response formats will be -;; defined in the future. A DoH server MUST be able to process -;; "application/dns-message" request messages. - -;; Each DNS request-response pair is mapped to one HTTP exchange. - -;; DNS response codes indicate either success or failure for the DNS -;; query. A successful HTTP response with a 2xx status code (see -;; Section 6.3 of [RFC7231]) is used for any valid DNS response, - -;; HTTP responses with non-successful HTTP status codes do not contain -;; replies to the original DNS question in the HTTP request. DoH -;; clients need to use the same semantic processing of non-successful -;; HTTP status codes as other HTTP clients. - -;; 4.2.2. HTTP Response Example - -;; This is an example response for a query for the IN AAAA records for -;; "www.example.com" with recursion turned on. The response bears one -;; answer record with an address of 2001:db8:abcd:12:1:2:3:4 and a TTL -;; of 3709 seconds. - -;; :status = 200 -;; content-type = application/dns-message -;; content-length = 61 -;; cache-control = max-age=3709 - -;; <61 bytes represented by the following hex encoding> -;; 00 00 81 80 00 01 00 01 00 00 00 00 03 77 77 77 -;; 07 65 78 61 6d 70 6c 65 03 63 6f 6d 00 00 1c 00 -;; 01 c0 0c 00 1c 00 01 00 00 0e 7d 00 10 20 01 0d -;; b8 ab cd 00 12 00 01 00 02 00 03 00 04 - -;; This protocol MUST be used with the https URI scheme [RFC7230]. - -;; In particular, DoH servers SHOULD assign an explicit HTTP freshness -;; lifetime (see Section 4.2 of [RFC7234]) so that the DoH client is -;; more likely to use fresh DNS data. This requirement is due to HTTP -;; caches being able to assign their own heuristic freshness (such as -;; that described in Section 4.2.2 of [RFC7234]), which would take -;; control of the cache contents out of the hands of the DoH server. - - -;; The assigned freshness lifetime of a DoH HTTP response MUST be less -;; than or equal to the smallest TTL in the Answer section of the DNS -;; response. A freshness lifetime equal to the smallest TTL in the -;; Answer section is RECOMMENDED. For example, if a HTTP response -;; carries three RRsets with TTLs of 30, 600, and 300, the HTTP -;; freshness lifetime should be 30 seconds (which could be specified as -;; "Cache-Control: max-age=30"). This requirement helps prevent expired -;; RRsets in messages in an HTTP cache from unintentionally being -;; served. - -;; If the DNS response has no records in the Answer section, and the DNS -;; response has an SOA record in the Authority section, the response -;; freshness lifetime MUST NOT be greater than the MINIMUM field from -;; that SOA record (see [RFC2308]). - -;; DoH clients MUST account for the Age response header field's value -;; [RFC7234] when calculating the DNS TTL of a response. For example, -;; if an RRset is received with a DNS TTL of 600, but the Age header -;; field indicates that the response has been cached for 250 seconds, -;; the remaining lifetime of the RRset is 350 seconds. This requirement -;; applies to both DoH client HTTP caches and DoH client DNS caches. - -;; Those features were introduced to HTTP in HTTP/2 [RFC7540]. -;; Earlier versions of HTTP are capable of conveying the semantic -;; requirements of DoH but may result in very poor performance. - -;; In order to maximize interoperability, DoH clients and DoH servers -;; MUST support the "application/dns-message" media type. - -;; The data payload for the "application/dns-message" media type is a -;; single message of the DNS on-the-wire format defined in Section 4.2.1 -;; of [RFC1035], which in turn refers to the full wire format defined in -;; Section 4.1 of that RFC. - -;; This media type restricts the maximum size of the DNS message to -;; 65535 bytes. - -;; When using the GET method, the data payload for this media type MUST -;; be encoded with base64url [RFC4648] and then provided as a variable -;; named "dns" to the URI Template expansion. Padding characters for -;; base64url MUST NOT be included. - -;; When using the POST method, the data payload for this media type MUST -;; NOT be encoded and is used directly as the HTTP message body. From 72abb43577e3c80acf6dfaab4a3de080476e8803 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 26 Jan 2020 20:00:22 +0000 Subject: [PATCH 6/8] chore(lisp/dns): Add 'message.lisp' to build instructions --- lisp/dns/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/lisp/dns/default.nix b/lisp/dns/default.nix index 134540b2b..242608658 100644 --- a/lisp/dns/default.nix +++ b/lisp/dns/default.nix @@ -13,6 +13,7 @@ pkgs.nix.buildLisp.library { srcs = [ ./package.lisp + ./message.lisp ./client.lisp ]; } From 4c109f66b6c1a859ca7b96954e13883e38751350 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 26 Jan 2020 20:29:30 +0000 Subject: [PATCH 7/8] feat(lisp/dns): Introduce enum for DNS types & decode RDATA Adds some of the most common DNS types in the enum (others TBD), and starts decoding RDATA for TXT and A. --- lisp/dns/message.lisp | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/lisp/dns/message.lisp b/lisp/dns/message.lisp index 0eedbe660..137532b74 100644 --- a/lisp/dns/message.lisp +++ b/lisp/dns/message.lisp @@ -662,6 +662,21 @@ ;; 4.1.3. Resource record format (defbinary dns-rr (:byte-order :big-endian) +(define-enum dns-type 2 + (:byte-order :big-endian) + + ;; http://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml + (A 1) + (NS 2) + (CNAME 5) + (SOA 6) + (PTR 12) + (MX 15) + (TXT 16) + (SRV 33) + (AAAA 28) + (ANY 255)) ;; (typically wants SOA, MX, NS and MX) + (name nil :type (custom :lisp-type qname :reader #'read-qname :writer #'write-qname)) @@ -669,7 +684,7 @@ ;; two octets containing one of the RR type codes. This ;; field specifies the meaning of the data in the RDATA ;; field. - (type 0 :type 16) ; TODO(tazjin): enum? + (type 0 :type dns-type) ;; two octets which specify the class of the data in the ;; RDATA field. @@ -691,7 +706,11 @@ ;; according to the TYPE and CLASS of the resource record. ;; For example, the if the TYPE is A and the CLASS is IN, ;; the RDATA field is a 4 octet ARPA Internet address. - (rdata #() :type (simple-array (unsigned-byte 8) (rdlength)))) + (rdata #() :type (eval (case type + ;; TODO(tazjin): Deal with multiple strings in single RRDATA + ((TXT) '(counted-string 1)) + ((A) '(simple-array (unsigned-byte 8) (4))) + (otherwise `(simple-array (unsigned-byte 8) (,rdlength))))))) (defbinary dns-message (:byte-order :big-endian) (header nil :type dns-header) From 3f9546197e11357ec7c62d225ed2d1820a22ce2f Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 26 Jan 2020 20:34:03 +0000 Subject: [PATCH 8/8] feat(lisp/dns): Export struct fields --- lisp/dns/message.lisp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lisp/dns/message.lisp b/lisp/dns/message.lisp index 137532b74..d4120da38 100644 --- a/lisp/dns/message.lisp +++ b/lisp/dns/message.lisp @@ -643,7 +643,7 @@ (write-byte 0 stream)) ;; 4.1.2. Question section format -(defbinary dns-question (:byte-order :big-endian) +(defbinary dns-question (:byte-order :big-endian :export t) ;; a domain name represented (qname "" :type (custom :lisp-type qname :reader #'read-qname @@ -661,7 +661,6 @@ ;; 4.1.3. Resource record format -(defbinary dns-rr (:byte-order :big-endian) (define-enum dns-type 2 (:byte-order :big-endian) @@ -677,6 +676,7 @@ (AAAA 28) (ANY 255)) ;; (typically wants SOA, MX, NS and MX) +(defbinary dns-rr (:byte-order :big-endian :export t) (name nil :type (custom :lisp-type qname :reader #'read-qname :writer #'write-qname)) @@ -712,7 +712,7 @@ ((A) '(simple-array (unsigned-byte 8) (4))) (otherwise `(simple-array (unsigned-byte 8) (,rdlength))))))) -(defbinary dns-message (:byte-order :big-endian) +(defbinary dns-message (:byte-order :big-endian :export t) (header nil :type dns-header) ;; the question for the name server