From 371a444eb776873848868819e9e7a08c7d810947 Mon Sep 17 00:00:00 2001 From: Griffin Smith Date: Fri, 24 Dec 2021 12:10:54 -0500 Subject: [PATCH] feat(grfn/bbbg): Allow importing event attendees Add support for importing the tsv that meetup exports into a list of upserted attendees, and event-attendee joins. Change-Id: I5f4ddc9fc63bcc6b0334bc3e1d3cbc4d5b99c21b Reviewed-on: https://cl.tvl.fyi/c/depot/+/4570 Tested-by: BuildkiteCI Reviewed-by: grfn Autosubmit: grfn --- users/grfn/bbbg/deps.edn | 4 +- users/grfn/bbbg/deps.nix | 13 ++ ...028-add-attendee-unique-meetup-id.down.sql | 1 + ...61028-add-attendee-unique-meetup-id.up.sql | 2 + users/grfn/bbbg/src/bbbg/attendee.clj | 4 +- users/grfn/bbbg/src/bbbg/db.clj | 3 +- users/grfn/bbbg/src/bbbg/db/attendee.clj | 16 ++- .../grfn/bbbg/src/bbbg/db/attendee_check.clj | 47 +++---- .../grfn/bbbg/src/bbbg/db/event_attendee.clj | 16 +++ users/grfn/bbbg/src/bbbg/event_attendee.clj | 2 + users/grfn/bbbg/src/bbbg/meetup/import.clj | 124 ++++++++++++++++++ users/grfn/bbbg/src/bbbg/meetup_user.clj | 8 ++ users/grfn/bbbg/src/bbbg/util/core.clj | 8 ++ users/grfn/bbbg/src/bbbg/util/spec.clj | 16 +++ .../bbbg/test/bbbg/meetup/import_test.clj | 7 + 15 files changed, 242 insertions(+), 29 deletions(-) create mode 100644 users/grfn/bbbg/resources/migrations/20211224161028-add-attendee-unique-meetup-id.down.sql create mode 100644 users/grfn/bbbg/resources/migrations/20211224161028-add-attendee-unique-meetup-id.up.sql create mode 100644 users/grfn/bbbg/src/bbbg/db/event_attendee.clj create mode 100644 users/grfn/bbbg/src/bbbg/meetup/import.clj create mode 100644 users/grfn/bbbg/src/bbbg/meetup_user.clj create mode 100644 users/grfn/bbbg/src/bbbg/util/spec.clj create mode 100644 users/grfn/bbbg/test/bbbg/meetup/import_test.clj diff --git a/users/grfn/bbbg/deps.edn b/users/grfn/bbbg/deps.edn index 433d0c64f..6d05fa2a1 100644 --- a/users/grfn/bbbg/deps.edn +++ b/users/grfn/bbbg/deps.edn @@ -21,8 +21,6 @@ hiccup/hiccup {:mvn/version "1.0.5"} garden/garden {:mvn/version "1.3.10"} - ;; Utils - com.stuartsierra/component {:mvn/version "1.0.0"} ;; Logging + Observability ch.qos.logback/logback-classic {:mvn/version "1.2.3" @@ -37,10 +35,12 @@ clj-commons/iapetos {:mvn/version "0.1.12"} ;; Utilities + com.stuartsierra/component {:mvn/version "1.0.0"} yogthos/config {:mvn/version "1.1.8"} clojure.java-time/clojure.java-time {:mvn/version "0.3.3"} cheshire/cheshire {:mvn/version "5.10.1"} org.apache.commons/commons-lang3 {:mvn/version "3.11"} + org.clojure/data.csv {:mvn/version "1.0.0"} ;; Spec org.clojure/spec.alpha {:mvn/version "0.3.214"} diff --git a/users/grfn/bbbg/deps.nix b/users/grfn/bbbg/deps.nix index 8bf2ad5ab..0a55932ad 100644 --- a/users/grfn/bbbg/deps.nix +++ b/users/grfn/bbbg/deps.nix @@ -847,6 +847,19 @@ let repos = [ paths = [ src ]; } + rec { + name = "data.csv/org.clojure"; + src = fetchMavenArtifact { + inherit repos; + artifactId = "data.csv"; + groupId = "org.clojure"; + sha512 = "b039775a859ed27eca8f8ae74ccb6afde3ad1fe2b3cbe542240c324d60fe1237e495eb1300ee9eb4ff4ef59f01faf7aec6ef1dd6a025ee4fe556c1d91acfcf1b"; + version = "1.0.0"; + + }; + paths = [ src ]; + } + rec { name = "simpleclient_tracer_otel_agent/io.prometheus"; src = fetchMavenArtifact { diff --git a/users/grfn/bbbg/resources/migrations/20211224161028-add-attendee-unique-meetup-id.down.sql b/users/grfn/bbbg/resources/migrations/20211224161028-add-attendee-unique-meetup-id.down.sql new file mode 100644 index 000000000..cbee0c00a --- /dev/null +++ b/users/grfn/bbbg/resources/migrations/20211224161028-add-attendee-unique-meetup-id.down.sql @@ -0,0 +1 @@ +drop index attendee_uniq_meetup_user_id; diff --git a/users/grfn/bbbg/resources/migrations/20211224161028-add-attendee-unique-meetup-id.up.sql b/users/grfn/bbbg/resources/migrations/20211224161028-add-attendee-unique-meetup-id.up.sql new file mode 100644 index 000000000..5895cad56 --- /dev/null +++ b/users/grfn/bbbg/resources/migrations/20211224161028-add-attendee-unique-meetup-id.up.sql @@ -0,0 +1,2 @@ +create unique index "attendee_uniq_meetup_user_id" on attendee (meetup_user_id); +-- ;; diff --git a/users/grfn/bbbg/src/bbbg/attendee.clj b/users/grfn/bbbg/src/bbbg/attendee.clj index 6088e1f03..49a6d621d 100644 --- a/users/grfn/bbbg/src/bbbg/attendee.clj +++ b/users/grfn/bbbg/src/bbbg/attendee.clj @@ -3,10 +3,8 @@ (s/def ::id uuid?) -(s/def ::meetup-name string?) +(s/def ::meetup-name (s/and string? seq)) (s/def ::discord-name (s/nilable string?)) -(s/def ::meetup-user-id (s/nilable string?)) - (s/def ::organizer-notes string?) diff --git a/users/grfn/bbbg/src/bbbg/db.clj b/users/grfn/bbbg/src/bbbg/db.clj index f10cae3b8..3ac3d962c 100644 --- a/users/grfn/bbbg/src/bbbg/db.clj +++ b/users/grfn/bbbg/src/bbbg/db.clj @@ -355,6 +355,7 @@ (comment (def db (:db bbbg.core/system)) - (generate-migration db "add-attendee-checks") + (generate-migration db "add-attendee-unique-meetup-id") (migrate! db) + ) diff --git a/users/grfn/bbbg/src/bbbg/db/attendee.clj b/users/grfn/bbbg/src/bbbg/db/attendee.clj index 06f495c57..089b92457 100644 --- a/users/grfn/bbbg/src/bbbg/db/attendee.clj +++ b/users/grfn/bbbg/src/bbbg/db/attendee.clj @@ -6,7 +6,8 @@ honeysql-postgres.helpers [honeysql.helpers :refer - [merge-group-by merge-join merge-left-join merge-select merge-where]])) + [merge-group-by merge-join merge-left-join merge-select merge-where]] + [bbbg.util.core :as u])) (defn search ([q] (search {:select [:attendee.*] :from [:attendee]} q)) @@ -41,6 +42,19 @@ [:not :event_attendee.attended]]) :no-shows])))) +(defn upsert-all! + [db attendees] + (db/list + db + {:insert-into :attendee + :values (map #(->> % + (db/process-key-map :attendee) + (u/map-keys keyword)) + attendees) + :upsert {:on-conflict [:meetup-user-id] + :do-update-set [:meetup-name]} + :returning [:id :meetup-user-id]})) + (comment (def db (:db bbbg.core/system)) (db/database? db) diff --git a/users/grfn/bbbg/src/bbbg/db/attendee_check.clj b/users/grfn/bbbg/src/bbbg/db/attendee_check.clj index 66090b16c..33ed98d4f 100644 --- a/users/grfn/bbbg/src/bbbg/db/attendee_check.clj +++ b/users/grfn/bbbg/src/bbbg/db/attendee_check.clj @@ -8,28 +8,31 @@ (defn attendees-with-last-checks [db attendees] - (let [ids (map ::attendee/id attendees) - checks - (db/list db {:select [:attendee-check.*] - :from [:attendee-check] - :join [[{:select [:%max.attendee-check.checked-at - :attendee-check.attendee-id] - :from [:attendee-check] - :group-by [:attendee-check.attendee-id] - :where [:in :attendee-check.attendee-id ids]} - :last-check] - [:= - :attendee-check.attendee-id - :last-check.attendee-id]]}) - users (u/key-by - ::user/id - (db/list db {:select [:public.user.*] - :from [:public.user] - :where [:in :id (map ::user/id checks)]})) - checks (map #(assoc % :user (users (::user/id %))) checks) - attendee-id->check (u/key-by ::attendee/id checks)] - (map #(assoc % :last-check (attendee-id->check (::attendee/id %))) - attendees))) + (when (seq attendees) + (let [ids (map ::attendee/id attendees) + checks + (db/list db {:select [:attendee-check.*] + :from [:attendee-check] + :join [[{:select [:%max.attendee-check.checked-at + :attendee-check.attendee-id] + :from [:attendee-check] + :group-by [:attendee-check.attendee-id] + :where [:in :attendee-check.attendee-id ids]} + :last-check] + [:= + :attendee-check.attendee-id + :last-check.attendee-id]]}) + users (if (seq checks) + (u/key-by + ::user/id + (db/list db {:select [:public.user.*] + :from [:public.user] + :where [:in :id (map ::user/id checks)]})) + {}) + checks (map #(assoc % :user (users (::user/id %))) checks) + attendee-id->check (u/key-by ::attendee/id checks)] + (map #(assoc % :last-check (attendee-id->check (::attendee/id %))) + attendees)))) (comment (def db (:db bbbg.core/system)) diff --git a/users/grfn/bbbg/src/bbbg/db/event_attendee.clj b/users/grfn/bbbg/src/bbbg/db/event_attendee.clj new file mode 100644 index 000000000..9fa8ad479 --- /dev/null +++ b/users/grfn/bbbg/src/bbbg/db/event_attendee.clj @@ -0,0 +1,16 @@ +(ns bbbg.db.event-attendee + (:require honeysql-postgres.format + [bbbg.db :as db] + [bbbg.util.core :as u])) + +(defn upsert-all! + [db attendees] + (db/execute! + db + {:insert-into :event-attendee + :values (map #(->> % + (db/process-key-map :event-attendee) + (u/map-keys keyword)) + attendees) + :upsert {:on-conflict [:event-id :attendee-id] + :do-update-set [:rsvpd-attending]}})) diff --git a/users/grfn/bbbg/src/bbbg/event_attendee.clj b/users/grfn/bbbg/src/bbbg/event_attendee.clj index af37bf01c..7b6b4c276 100644 --- a/users/grfn/bbbg/src/bbbg/event_attendee.clj +++ b/users/grfn/bbbg/src/bbbg/event_attendee.clj @@ -2,3 +2,5 @@ (:require [clojure.spec.alpha :as s])) (s/def ::attended? boolean?) + +(s/def ::rsvpd-attending? boolean?) diff --git a/users/grfn/bbbg/src/bbbg/meetup/import.clj b/users/grfn/bbbg/src/bbbg/meetup/import.clj new file mode 100644 index 000000000..29aaf53e5 --- /dev/null +++ b/users/grfn/bbbg/src/bbbg/meetup/import.clj @@ -0,0 +1,124 @@ +(ns bbbg.meetup.import + (:require + [bbbg.attendee :as attendee] + [bbbg.db.attendee :as db.attendee] + [bbbg.db.event-attendee :as db.event-attendee] + [bbbg.event :as event] + [bbbg.event-attendee :as event-attendee] + [bbbg.meetup-user :as meetup-user] + [bbbg.util.core :as u] + [bbbg.util.spec :as u.s] + [clojure.data.csv :as csv] + [clojure.java.io :as io] + [clojure.spec.alpha :as s] + [clojure.string :as str] + [expound.alpha :as exp])) + +(def spreadsheet-column->key + {"Name" :name + "User ID" :user-id + "Title" :title + "Event Host" :event-host + "RSVP" :rsvp + "Guests" :guests + "RSVPed on" :rsvped-on + "Joined Group on" :joined-group-on + "URL of Member Profile" :member-profile-url}) + +(defn read-attendees [f] + (with-open [reader (io/reader f)] + (let [[headers & rows] (-> reader (csv/read-csv :separator \tab)) + keys (map spreadsheet-column->key headers)] + (doall + (->> rows + (map (partial zipmap keys)) + (map (partial u/filter-kv (fn [k _] (some? k)))) + (filter (partial some (comp seq val)))))))) + +;;; + +(s/def ::imported-attendee + (s/keys :req [::attendee/meetup-name + ::meetup-user/id])) + +(def key->attendee-col + {:name ::attendee/meetup-name + :user-id ::meetup-user/id}) + +(defn row-user-id->user-id [row-id] + (str/replace-first row-id "user " "")) + +(defn check-attendee [attendee] + () + (if (s/valid? ::imported-attendee attendee) + attendee + (throw (ex-info + (str "Invalid imported attendee\n" + (exp/expound-str ::imported-attendee attendee)) + (assoc (s/explain-data ::imported-attendee attendee) + ::s/failure + ::s/assertion-failed))))) + +(defn row->attendee [r] + (u.s/assert! + ::imported-attendee + (update (u/keep-keys key->attendee-col r) + ::meetup-user/id row-user-id->user-id))) + +;;; + +(s/def ::imported-event-attendee + (s/keys :req [::event-attendee/rsvpd-attending? + ::attendee/id + ::event/id])) + +(def key->event-attendee-col + {:rsvp ::event-attendee/rsvpd-attending?}) + +(defn row->event-attendee + [{event-id ::event/id :keys [meetup-id->attendee-id]} r] + (let [attendee-id (-> r :user-id row-user-id->user-id meetup-id->attendee-id)] + (u.s/assert! + ::imported-event-attendee + (-> (u/keep-keys key->event-attendee-col r) + (update ::event-attendee/rsvpd-attending? + (partial = "Yes")) + (assoc ::event/id event-id + ::attendee/id attendee-id))))) + +;;; + +(defn import-data! [db event-id f] + (let [rows (read-attendees f) + attendees (db.attendee/upsert-all! db (map row->attendee rows)) + meetup-id->attendee-id (into {} + (map (juxt ::meetup-user/id ::attendee/id)) + attendees)] + (db.event-attendee/upsert-all! + db + (map (partial row->event-attendee + {::event/id event-id + :meetup-id->attendee-id meetup-id->attendee-id}) + rows)))) + +;;; Spreadsheet columns: +;;; +;;; Name +;;; User ID +;;; Title +;;; Event Host +;;; RSVP +;;; Guests +;;; RSVPed on +;;; Joined Group on +;;; URL of Member Profile +;;; Have you been to one of our events before? Note, attendance at all events will require proof of vaccination until further notice. + +(comment + (def -filename- "/home/grfn/code/depot/users/grfn/bbbg/sample-data.tsv") + (def event-id #uuid "09f8fed6-7480-451b-89a2-bb4edaeae657") + + (read-attendees -filename-) + (import-data! (:db bbbg.core/system) event-id -filename-) + + ) diff --git a/users/grfn/bbbg/src/bbbg/meetup_user.clj b/users/grfn/bbbg/src/bbbg/meetup_user.clj new file mode 100644 index 000000000..bd183132b --- /dev/null +++ b/users/grfn/bbbg/src/bbbg/meetup_user.clj @@ -0,0 +1,8 @@ +(ns bbbg.meetup-user + (:require [clojure.spec.alpha :as s])) + +(s/def ::id + (s/nilable + (s/and string? + seq + #(re-matches #"\d+" %)))) diff --git a/users/grfn/bbbg/src/bbbg/util/core.clj b/users/grfn/bbbg/src/bbbg/util/core.clj index 9ef8ef6be..d458aa559 100644 --- a/users/grfn/bbbg/src/bbbg/util/core.clj +++ b/users/grfn/bbbg/src/bbbg/util/core.clj @@ -52,6 +52,14 @@ ([f] (map-kv f identity)) ([f m] (map-kv f identity m))) +(defn keep-keys + "Map f over the keys of m, keeping only those entries for which f does not + return nil. Preserves metadata on the incoming map. The one-argument form + returns a transducer that yields map-entries." + ([f] (keep (fn [[k v]] (when-let [k' (f k)] + (first {k' v}))))) + ([f m] (into (empty m) (keep-keys f) m))) + (defn map-vals "Map f over the values of m. Preserves metadata on the incoming map. The one-argument form returns a transducer that yields map-entries." diff --git a/users/grfn/bbbg/src/bbbg/util/spec.clj b/users/grfn/bbbg/src/bbbg/util/spec.clj new file mode 100644 index 000000000..89ac92669 --- /dev/null +++ b/users/grfn/bbbg/src/bbbg/util/spec.clj @@ -0,0 +1,16 @@ +(ns bbbg.util.spec + (:require [expound.alpha :as exp] + [clojure.spec.alpha :as s])) + +(defn assert! + ([spec s] (assert! "Spec assertion failed" spec s)) + ([message spec x] + (if (s/valid? spec x) + x + (throw (ex-info + (str message + "\n" + (exp/expound-str spec x)) + (assoc (s/explain-data spec x) + ::s/failure + ::s/assertion-failed)))))) diff --git a/users/grfn/bbbg/test/bbbg/meetup/import_test.clj b/users/grfn/bbbg/test/bbbg/meetup/import_test.clj new file mode 100644 index 000000000..d7d698a58 --- /dev/null +++ b/users/grfn/bbbg/test/bbbg/meetup/import_test.clj @@ -0,0 +1,7 @@ +(ns bbbg.meetup.import-test + (:require [bbbg.meetup.import :as sut] + [clojure.test :refer :all])) + +(deftest test-row-user-id->user-id + (is (= "246364067" (sut/row-user-id->user-id "user 246364067"))) + (is (= "246364067" (sut/row-user-id->user-id "246364067"))))