feat(grfn/bbbg): Allow importing event attendees

Add support for importing the tsv that meetup exports into a list of
upserted attendees, and event-attendee joins.

Change-Id: I5f4ddc9fc63bcc6b0334bc3e1d3cbc4d5b99c21b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/4570
Tested-by: BuildkiteCI
Reviewed-by: grfn <grfn@gws.fyi>
Autosubmit: grfn <grfn@gws.fyi>
This commit is contained in:
Griffin Smith 2021-12-24 12:10:54 -05:00 committed by clbot
parent 92d9580df3
commit 371a444eb7
15 changed files with 242 additions and 29 deletions

View file

@ -21,8 +21,6 @@
hiccup/hiccup {:mvn/version "1.0.5"} hiccup/hiccup {:mvn/version "1.0.5"}
garden/garden {:mvn/version "1.3.10"} garden/garden {:mvn/version "1.3.10"}
;; Utils
com.stuartsierra/component {:mvn/version "1.0.0"}
;; Logging + Observability ;; Logging + Observability
ch.qos.logback/logback-classic {:mvn/version "1.2.3" ch.qos.logback/logback-classic {:mvn/version "1.2.3"
@ -37,10 +35,12 @@
clj-commons/iapetos {:mvn/version "0.1.12"} clj-commons/iapetos {:mvn/version "0.1.12"}
;; Utilities ;; Utilities
com.stuartsierra/component {:mvn/version "1.0.0"}
yogthos/config {:mvn/version "1.1.8"} yogthos/config {:mvn/version "1.1.8"}
clojure.java-time/clojure.java-time {:mvn/version "0.3.3"} clojure.java-time/clojure.java-time {:mvn/version "0.3.3"}
cheshire/cheshire {:mvn/version "5.10.1"} cheshire/cheshire {:mvn/version "5.10.1"}
org.apache.commons/commons-lang3 {:mvn/version "3.11"} org.apache.commons/commons-lang3 {:mvn/version "3.11"}
org.clojure/data.csv {:mvn/version "1.0.0"}
;; Spec ;; Spec
org.clojure/spec.alpha {:mvn/version "0.3.214"} org.clojure/spec.alpha {:mvn/version "0.3.214"}

View file

@ -847,6 +847,19 @@ let repos = [
paths = [ src ]; paths = [ src ];
} }
rec {
name = "data.csv/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "data.csv";
groupId = "org.clojure";
sha512 = "b039775a859ed27eca8f8ae74ccb6afde3ad1fe2b3cbe542240c324d60fe1237e495eb1300ee9eb4ff4ef59f01faf7aec6ef1dd6a025ee4fe556c1d91acfcf1b";
version = "1.0.0";
};
paths = [ src ];
}
rec { rec {
name = "simpleclient_tracer_otel_agent/io.prometheus"; name = "simpleclient_tracer_otel_agent/io.prometheus";
src = fetchMavenArtifact { src = fetchMavenArtifact {

View file

@ -0,0 +1 @@
drop index attendee_uniq_meetup_user_id;

View file

@ -0,0 +1,2 @@
create unique index "attendee_uniq_meetup_user_id" on attendee (meetup_user_id);
-- ;;

View file

@ -3,10 +3,8 @@
(s/def ::id uuid?) (s/def ::id uuid?)
(s/def ::meetup-name string?) (s/def ::meetup-name (s/and string? seq))
(s/def ::discord-name (s/nilable string?)) (s/def ::discord-name (s/nilable string?))
(s/def ::meetup-user-id (s/nilable string?))
(s/def ::organizer-notes string?) (s/def ::organizer-notes string?)

View file

@ -355,6 +355,7 @@
(comment (comment
(def db (:db bbbg.core/system)) (def db (:db bbbg.core/system))
(generate-migration db "add-attendee-checks") (generate-migration db "add-attendee-unique-meetup-id")
(migrate! db) (migrate! db)
) )

View file

@ -6,7 +6,8 @@
honeysql-postgres.helpers honeysql-postgres.helpers
[honeysql.helpers [honeysql.helpers
:refer :refer
[merge-group-by merge-join merge-left-join merge-select merge-where]])) [merge-group-by merge-join merge-left-join merge-select merge-where]]
[bbbg.util.core :as u]))
(defn search (defn search
([q] (search {:select [:attendee.*] :from [:attendee]} q)) ([q] (search {:select [:attendee.*] :from [:attendee]} q))
@ -41,6 +42,19 @@
[:not :event_attendee.attended]]) [:not :event_attendee.attended]])
:no-shows])))) :no-shows]))))
(defn upsert-all!
[db attendees]
(db/list
db
{:insert-into :attendee
:values (map #(->> %
(db/process-key-map :attendee)
(u/map-keys keyword))
attendees)
:upsert {:on-conflict [:meetup-user-id]
:do-update-set [:meetup-name]}
:returning [:id :meetup-user-id]}))
(comment (comment
(def db (:db bbbg.core/system)) (def db (:db bbbg.core/system))
(db/database? db) (db/database? db)

View file

@ -8,28 +8,31 @@
(defn attendees-with-last-checks (defn attendees-with-last-checks
[db attendees] [db attendees]
(let [ids (map ::attendee/id attendees) (when (seq attendees)
checks (let [ids (map ::attendee/id attendees)
(db/list db {:select [:attendee-check.*] checks
:from [:attendee-check] (db/list db {:select [:attendee-check.*]
:join [[{:select [:%max.attendee-check.checked-at :from [:attendee-check]
:attendee-check.attendee-id] :join [[{:select [:%max.attendee-check.checked-at
:from [:attendee-check] :attendee-check.attendee-id]
:group-by [:attendee-check.attendee-id] :from [:attendee-check]
:where [:in :attendee-check.attendee-id ids]} :group-by [:attendee-check.attendee-id]
:last-check] :where [:in :attendee-check.attendee-id ids]}
[:= :last-check]
:attendee-check.attendee-id [:=
:last-check.attendee-id]]}) :attendee-check.attendee-id
users (u/key-by :last-check.attendee-id]]})
::user/id users (if (seq checks)
(db/list db {:select [:public.user.*] (u/key-by
:from [:public.user] ::user/id
:where [:in :id (map ::user/id checks)]})) (db/list db {:select [:public.user.*]
checks (map #(assoc % :user (users (::user/id %))) checks) :from [:public.user]
attendee-id->check (u/key-by ::attendee/id checks)] :where [:in :id (map ::user/id checks)]}))
(map #(assoc % :last-check (attendee-id->check (::attendee/id %))) {})
attendees))) checks (map #(assoc % :user (users (::user/id %))) checks)
attendee-id->check (u/key-by ::attendee/id checks)]
(map #(assoc % :last-check (attendee-id->check (::attendee/id %)))
attendees))))
(comment (comment
(def db (:db bbbg.core/system)) (def db (:db bbbg.core/system))

View file

@ -0,0 +1,16 @@
(ns bbbg.db.event-attendee
(:require honeysql-postgres.format
[bbbg.db :as db]
[bbbg.util.core :as u]))
(defn upsert-all!
[db attendees]
(db/execute!
db
{:insert-into :event-attendee
:values (map #(->> %
(db/process-key-map :event-attendee)
(u/map-keys keyword))
attendees)
:upsert {:on-conflict [:event-id :attendee-id]
:do-update-set [:rsvpd-attending]}}))

View file

@ -2,3 +2,5 @@
(:require [clojure.spec.alpha :as s])) (:require [clojure.spec.alpha :as s]))
(s/def ::attended? boolean?) (s/def ::attended? boolean?)
(s/def ::rsvpd-attending? boolean?)

View file

@ -0,0 +1,124 @@
(ns bbbg.meetup.import
(:require
[bbbg.attendee :as attendee]
[bbbg.db.attendee :as db.attendee]
[bbbg.db.event-attendee :as db.event-attendee]
[bbbg.event :as event]
[bbbg.event-attendee :as event-attendee]
[bbbg.meetup-user :as meetup-user]
[bbbg.util.core :as u]
[bbbg.util.spec :as u.s]
[clojure.data.csv :as csv]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[clojure.string :as str]
[expound.alpha :as exp]))
(def spreadsheet-column->key
{"Name" :name
"User ID" :user-id
"Title" :title
"Event Host" :event-host
"RSVP" :rsvp
"Guests" :guests
"RSVPed on" :rsvped-on
"Joined Group on" :joined-group-on
"URL of Member Profile" :member-profile-url})
(defn read-attendees [f]
(with-open [reader (io/reader f)]
(let [[headers & rows] (-> reader (csv/read-csv :separator \tab))
keys (map spreadsheet-column->key headers)]
(doall
(->> rows
(map (partial zipmap keys))
(map (partial u/filter-kv (fn [k _] (some? k))))
(filter (partial some (comp seq val))))))))
;;;
(s/def ::imported-attendee
(s/keys :req [::attendee/meetup-name
::meetup-user/id]))
(def key->attendee-col
{:name ::attendee/meetup-name
:user-id ::meetup-user/id})
(defn row-user-id->user-id [row-id]
(str/replace-first row-id "user " ""))
(defn check-attendee [attendee]
()
(if (s/valid? ::imported-attendee attendee)
attendee
(throw (ex-info
(str "Invalid imported attendee\n"
(exp/expound-str ::imported-attendee attendee))
(assoc (s/explain-data ::imported-attendee attendee)
::s/failure
::s/assertion-failed)))))
(defn row->attendee [r]
(u.s/assert!
::imported-attendee
(update (u/keep-keys key->attendee-col r)
::meetup-user/id row-user-id->user-id)))
;;;
(s/def ::imported-event-attendee
(s/keys :req [::event-attendee/rsvpd-attending?
::attendee/id
::event/id]))
(def key->event-attendee-col
{:rsvp ::event-attendee/rsvpd-attending?})
(defn row->event-attendee
[{event-id ::event/id :keys [meetup-id->attendee-id]} r]
(let [attendee-id (-> r :user-id row-user-id->user-id meetup-id->attendee-id)]
(u.s/assert!
::imported-event-attendee
(-> (u/keep-keys key->event-attendee-col r)
(update ::event-attendee/rsvpd-attending?
(partial = "Yes"))
(assoc ::event/id event-id
::attendee/id attendee-id)))))
;;;
(defn import-data! [db event-id f]
(let [rows (read-attendees f)
attendees (db.attendee/upsert-all! db (map row->attendee rows))
meetup-id->attendee-id (into {}
(map (juxt ::meetup-user/id ::attendee/id))
attendees)]
(db.event-attendee/upsert-all!
db
(map (partial row->event-attendee
{::event/id event-id
:meetup-id->attendee-id meetup-id->attendee-id})
rows))))
;;; Spreadsheet columns:
;;;
;;; Name
;;; User ID
;;; Title
;;; Event Host
;;; RSVP
;;; Guests
;;; RSVPed on
;;; Joined Group on
;;; URL of Member Profile
;;; Have you been to one of our events before? Note, attendance at all events will require proof of vaccination until further notice.
(comment
(def -filename- "/home/grfn/code/depot/users/grfn/bbbg/sample-data.tsv")
(def event-id #uuid "09f8fed6-7480-451b-89a2-bb4edaeae657")
(read-attendees -filename-)
(import-data! (:db bbbg.core/system) event-id -filename-)
)

View file

@ -0,0 +1,8 @@
(ns bbbg.meetup-user
(:require [clojure.spec.alpha :as s]))
(s/def ::id
(s/nilable
(s/and string?
seq
#(re-matches #"\d+" %))))

View file

@ -52,6 +52,14 @@
([f] (map-kv f identity)) ([f] (map-kv f identity))
([f m] (map-kv f identity m))) ([f m] (map-kv f identity m)))
(defn keep-keys
"Map f over the keys of m, keeping only those entries for which f does not
return nil. Preserves metadata on the incoming map. The one-argument form
returns a transducer that yields map-entries."
([f] (keep (fn [[k v]] (when-let [k' (f k)]
(first {k' v})))))
([f m] (into (empty m) (keep-keys f) m)))
(defn map-vals (defn map-vals
"Map f over the values of m. Preserves metadata on the incoming map. The "Map f over the values of m. Preserves metadata on the incoming map. The
one-argument form returns a transducer that yields map-entries." one-argument form returns a transducer that yields map-entries."

View file

@ -0,0 +1,16 @@
(ns bbbg.util.spec
(:require [expound.alpha :as exp]
[clojure.spec.alpha :as s]))
(defn assert!
([spec s] (assert! "Spec assertion failed" spec s))
([message spec x]
(if (s/valid? spec x)
x
(throw (ex-info
(str message
"\n"
(exp/expound-str spec x))
(assoc (s/explain-data spec x)
::s/failure
::s/assertion-failed))))))

View file

@ -0,0 +1,7 @@
(ns bbbg.meetup.import-test
(:require [bbbg.meetup.import :as sut]
[clojure.test :refer :all]))
(deftest test-row-user-id->user-id
(is (= "246364067" (sut/row-user-id->user-id "user 246364067")))
(is (= "246364067" (sut/row-user-id->user-id "246364067"))))