feat(grfn/bbbg): Allow importing event attendees
Add support for importing the tsv that meetup exports into a list of upserted attendees, and event-attendee joins. Change-Id: I5f4ddc9fc63bcc6b0334bc3e1d3cbc4d5b99c21b Reviewed-on: https://cl.tvl.fyi/c/depot/+/4570 Tested-by: BuildkiteCI Reviewed-by: grfn <grfn@gws.fyi> Autosubmit: grfn <grfn@gws.fyi>
This commit is contained in:
parent
92d9580df3
commit
371a444eb7
15 changed files with 242 additions and 29 deletions
|
@ -21,8 +21,6 @@
|
|||
hiccup/hiccup {:mvn/version "1.0.5"}
|
||||
garden/garden {:mvn/version "1.3.10"}
|
||||
|
||||
;; Utils
|
||||
com.stuartsierra/component {:mvn/version "1.0.0"}
|
||||
|
||||
;; Logging + Observability
|
||||
ch.qos.logback/logback-classic {:mvn/version "1.2.3"
|
||||
|
@ -37,10 +35,12 @@
|
|||
clj-commons/iapetos {:mvn/version "0.1.12"}
|
||||
|
||||
;; Utilities
|
||||
com.stuartsierra/component {:mvn/version "1.0.0"}
|
||||
yogthos/config {:mvn/version "1.1.8"}
|
||||
clojure.java-time/clojure.java-time {:mvn/version "0.3.3"}
|
||||
cheshire/cheshire {:mvn/version "5.10.1"}
|
||||
org.apache.commons/commons-lang3 {:mvn/version "3.11"}
|
||||
org.clojure/data.csv {:mvn/version "1.0.0"}
|
||||
|
||||
;; Spec
|
||||
org.clojure/spec.alpha {:mvn/version "0.3.214"}
|
||||
|
|
|
@ -847,6 +847,19 @@ let repos = [
|
|||
paths = [ src ];
|
||||
}
|
||||
|
||||
rec {
|
||||
name = "data.csv/org.clojure";
|
||||
src = fetchMavenArtifact {
|
||||
inherit repos;
|
||||
artifactId = "data.csv";
|
||||
groupId = "org.clojure";
|
||||
sha512 = "b039775a859ed27eca8f8ae74ccb6afde3ad1fe2b3cbe542240c324d60fe1237e495eb1300ee9eb4ff4ef59f01faf7aec6ef1dd6a025ee4fe556c1d91acfcf1b";
|
||||
version = "1.0.0";
|
||||
|
||||
};
|
||||
paths = [ src ];
|
||||
}
|
||||
|
||||
rec {
|
||||
name = "simpleclient_tracer_otel_agent/io.prometheus";
|
||||
src = fetchMavenArtifact {
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
drop index attendee_uniq_meetup_user_id;
|
|
@ -0,0 +1,2 @@
|
|||
create unique index "attendee_uniq_meetup_user_id" on attendee (meetup_user_id);
|
||||
-- ;;
|
|
@ -3,10 +3,8 @@
|
|||
|
||||
(s/def ::id uuid?)
|
||||
|
||||
(s/def ::meetup-name string?)
|
||||
(s/def ::meetup-name (s/and string? seq))
|
||||
|
||||
(s/def ::discord-name (s/nilable string?))
|
||||
|
||||
(s/def ::meetup-user-id (s/nilable string?))
|
||||
|
||||
(s/def ::organizer-notes string?)
|
||||
|
|
|
@ -355,6 +355,7 @@
|
|||
|
||||
(comment
|
||||
(def db (:db bbbg.core/system))
|
||||
(generate-migration db "add-attendee-checks")
|
||||
(generate-migration db "add-attendee-unique-meetup-id")
|
||||
(migrate! db)
|
||||
|
||||
)
|
||||
|
|
|
@ -6,7 +6,8 @@
|
|||
honeysql-postgres.helpers
|
||||
[honeysql.helpers
|
||||
:refer
|
||||
[merge-group-by merge-join merge-left-join merge-select merge-where]]))
|
||||
[merge-group-by merge-join merge-left-join merge-select merge-where]]
|
||||
[bbbg.util.core :as u]))
|
||||
|
||||
(defn search
|
||||
([q] (search {:select [:attendee.*] :from [:attendee]} q))
|
||||
|
@ -41,6 +42,19 @@
|
|||
[:not :event_attendee.attended]])
|
||||
:no-shows]))))
|
||||
|
||||
(defn upsert-all!
|
||||
[db attendees]
|
||||
(db/list
|
||||
db
|
||||
{:insert-into :attendee
|
||||
:values (map #(->> %
|
||||
(db/process-key-map :attendee)
|
||||
(u/map-keys keyword))
|
||||
attendees)
|
||||
:upsert {:on-conflict [:meetup-user-id]
|
||||
:do-update-set [:meetup-name]}
|
||||
:returning [:id :meetup-user-id]}))
|
||||
|
||||
(comment
|
||||
(def db (:db bbbg.core/system))
|
||||
(db/database? db)
|
||||
|
|
|
@ -8,6 +8,7 @@
|
|||
|
||||
(defn attendees-with-last-checks
|
||||
[db attendees]
|
||||
(when (seq attendees)
|
||||
(let [ids (map ::attendee/id attendees)
|
||||
checks
|
||||
(db/list db {:select [:attendee-check.*]
|
||||
|
@ -21,15 +22,17 @@
|
|||
[:=
|
||||
:attendee-check.attendee-id
|
||||
:last-check.attendee-id]]})
|
||||
users (u/key-by
|
||||
users (if (seq checks)
|
||||
(u/key-by
|
||||
::user/id
|
||||
(db/list db {:select [:public.user.*]
|
||||
:from [:public.user]
|
||||
:where [:in :id (map ::user/id checks)]}))
|
||||
{})
|
||||
checks (map #(assoc % :user (users (::user/id %))) checks)
|
||||
attendee-id->check (u/key-by ::attendee/id checks)]
|
||||
(map #(assoc % :last-check (attendee-id->check (::attendee/id %)))
|
||||
attendees)))
|
||||
attendees))))
|
||||
|
||||
(comment
|
||||
(def db (:db bbbg.core/system))
|
||||
|
|
16
users/grfn/bbbg/src/bbbg/db/event_attendee.clj
Normal file
16
users/grfn/bbbg/src/bbbg/db/event_attendee.clj
Normal file
|
@ -0,0 +1,16 @@
|
|||
(ns bbbg.db.event-attendee
|
||||
(:require honeysql-postgres.format
|
||||
[bbbg.db :as db]
|
||||
[bbbg.util.core :as u]))
|
||||
|
||||
(defn upsert-all!
|
||||
[db attendees]
|
||||
(db/execute!
|
||||
db
|
||||
{:insert-into :event-attendee
|
||||
:values (map #(->> %
|
||||
(db/process-key-map :event-attendee)
|
||||
(u/map-keys keyword))
|
||||
attendees)
|
||||
:upsert {:on-conflict [:event-id :attendee-id]
|
||||
:do-update-set [:rsvpd-attending]}}))
|
|
@ -2,3 +2,5 @@
|
|||
(:require [clojure.spec.alpha :as s]))
|
||||
|
||||
(s/def ::attended? boolean?)
|
||||
|
||||
(s/def ::rsvpd-attending? boolean?)
|
||||
|
|
124
users/grfn/bbbg/src/bbbg/meetup/import.clj
Normal file
124
users/grfn/bbbg/src/bbbg/meetup/import.clj
Normal file
|
@ -0,0 +1,124 @@
|
|||
(ns bbbg.meetup.import
|
||||
(:require
|
||||
[bbbg.attendee :as attendee]
|
||||
[bbbg.db.attendee :as db.attendee]
|
||||
[bbbg.db.event-attendee :as db.event-attendee]
|
||||
[bbbg.event :as event]
|
||||
[bbbg.event-attendee :as event-attendee]
|
||||
[bbbg.meetup-user :as meetup-user]
|
||||
[bbbg.util.core :as u]
|
||||
[bbbg.util.spec :as u.s]
|
||||
[clojure.data.csv :as csv]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.spec.alpha :as s]
|
||||
[clojure.string :as str]
|
||||
[expound.alpha :as exp]))
|
||||
|
||||
(def spreadsheet-column->key
|
||||
{"Name" :name
|
||||
"User ID" :user-id
|
||||
"Title" :title
|
||||
"Event Host" :event-host
|
||||
"RSVP" :rsvp
|
||||
"Guests" :guests
|
||||
"RSVPed on" :rsvped-on
|
||||
"Joined Group on" :joined-group-on
|
||||
"URL of Member Profile" :member-profile-url})
|
||||
|
||||
(defn read-attendees [f]
|
||||
(with-open [reader (io/reader f)]
|
||||
(let [[headers & rows] (-> reader (csv/read-csv :separator \tab))
|
||||
keys (map spreadsheet-column->key headers)]
|
||||
(doall
|
||||
(->> rows
|
||||
(map (partial zipmap keys))
|
||||
(map (partial u/filter-kv (fn [k _] (some? k))))
|
||||
(filter (partial some (comp seq val))))))))
|
||||
|
||||
;;;
|
||||
|
||||
(s/def ::imported-attendee
|
||||
(s/keys :req [::attendee/meetup-name
|
||||
::meetup-user/id]))
|
||||
|
||||
(def key->attendee-col
|
||||
{:name ::attendee/meetup-name
|
||||
:user-id ::meetup-user/id})
|
||||
|
||||
(defn row-user-id->user-id [row-id]
|
||||
(str/replace-first row-id "user " ""))
|
||||
|
||||
(defn check-attendee [attendee]
|
||||
()
|
||||
(if (s/valid? ::imported-attendee attendee)
|
||||
attendee
|
||||
(throw (ex-info
|
||||
(str "Invalid imported attendee\n"
|
||||
(exp/expound-str ::imported-attendee attendee))
|
||||
(assoc (s/explain-data ::imported-attendee attendee)
|
||||
::s/failure
|
||||
::s/assertion-failed)))))
|
||||
|
||||
(defn row->attendee [r]
|
||||
(u.s/assert!
|
||||
::imported-attendee
|
||||
(update (u/keep-keys key->attendee-col r)
|
||||
::meetup-user/id row-user-id->user-id)))
|
||||
|
||||
;;;
|
||||
|
||||
(s/def ::imported-event-attendee
|
||||
(s/keys :req [::event-attendee/rsvpd-attending?
|
||||
::attendee/id
|
||||
::event/id]))
|
||||
|
||||
(def key->event-attendee-col
|
||||
{:rsvp ::event-attendee/rsvpd-attending?})
|
||||
|
||||
(defn row->event-attendee
|
||||
[{event-id ::event/id :keys [meetup-id->attendee-id]} r]
|
||||
(let [attendee-id (-> r :user-id row-user-id->user-id meetup-id->attendee-id)]
|
||||
(u.s/assert!
|
||||
::imported-event-attendee
|
||||
(-> (u/keep-keys key->event-attendee-col r)
|
||||
(update ::event-attendee/rsvpd-attending?
|
||||
(partial = "Yes"))
|
||||
(assoc ::event/id event-id
|
||||
::attendee/id attendee-id)))))
|
||||
|
||||
;;;
|
||||
|
||||
(defn import-data! [db event-id f]
|
||||
(let [rows (read-attendees f)
|
||||
attendees (db.attendee/upsert-all! db (map row->attendee rows))
|
||||
meetup-id->attendee-id (into {}
|
||||
(map (juxt ::meetup-user/id ::attendee/id))
|
||||
attendees)]
|
||||
(db.event-attendee/upsert-all!
|
||||
db
|
||||
(map (partial row->event-attendee
|
||||
{::event/id event-id
|
||||
:meetup-id->attendee-id meetup-id->attendee-id})
|
||||
rows))))
|
||||
|
||||
;;; Spreadsheet columns:
|
||||
;;;
|
||||
;;; Name
|
||||
;;; User ID
|
||||
;;; Title
|
||||
;;; Event Host
|
||||
;;; RSVP
|
||||
;;; Guests
|
||||
;;; RSVPed on
|
||||
;;; Joined Group on
|
||||
;;; URL of Member Profile
|
||||
;;; Have you been to one of our events before? Note, attendance at all events will require proof of vaccination until further notice.
|
||||
|
||||
(comment
|
||||
(def -filename- "/home/grfn/code/depot/users/grfn/bbbg/sample-data.tsv")
|
||||
(def event-id #uuid "09f8fed6-7480-451b-89a2-bb4edaeae657")
|
||||
|
||||
(read-attendees -filename-)
|
||||
(import-data! (:db bbbg.core/system) event-id -filename-)
|
||||
|
||||
)
|
8
users/grfn/bbbg/src/bbbg/meetup_user.clj
Normal file
8
users/grfn/bbbg/src/bbbg/meetup_user.clj
Normal file
|
@ -0,0 +1,8 @@
|
|||
(ns bbbg.meetup-user
|
||||
(:require [clojure.spec.alpha :as s]))
|
||||
|
||||
(s/def ::id
|
||||
(s/nilable
|
||||
(s/and string?
|
||||
seq
|
||||
#(re-matches #"\d+" %))))
|
|
@ -52,6 +52,14 @@
|
|||
([f] (map-kv f identity))
|
||||
([f m] (map-kv f identity m)))
|
||||
|
||||
(defn keep-keys
|
||||
"Map f over the keys of m, keeping only those entries for which f does not
|
||||
return nil. Preserves metadata on the incoming map. The one-argument form
|
||||
returns a transducer that yields map-entries."
|
||||
([f] (keep (fn [[k v]] (when-let [k' (f k)]
|
||||
(first {k' v})))))
|
||||
([f m] (into (empty m) (keep-keys f) m)))
|
||||
|
||||
(defn map-vals
|
||||
"Map f over the values of m. Preserves metadata on the incoming map. The
|
||||
one-argument form returns a transducer that yields map-entries."
|
||||
|
|
16
users/grfn/bbbg/src/bbbg/util/spec.clj
Normal file
16
users/grfn/bbbg/src/bbbg/util/spec.clj
Normal file
|
@ -0,0 +1,16 @@
|
|||
(ns bbbg.util.spec
|
||||
(:require [expound.alpha :as exp]
|
||||
[clojure.spec.alpha :as s]))
|
||||
|
||||
(defn assert!
|
||||
([spec s] (assert! "Spec assertion failed" spec s))
|
||||
([message spec x]
|
||||
(if (s/valid? spec x)
|
||||
x
|
||||
(throw (ex-info
|
||||
(str message
|
||||
"\n"
|
||||
(exp/expound-str spec x))
|
||||
(assoc (s/explain-data spec x)
|
||||
::s/failure
|
||||
::s/assertion-failed))))))
|
7
users/grfn/bbbg/test/bbbg/meetup/import_test.clj
Normal file
7
users/grfn/bbbg/test/bbbg/meetup/import_test.clj
Normal file
|
@ -0,0 +1,7 @@
|
|||
(ns bbbg.meetup.import-test
|
||||
(:require [bbbg.meetup.import :as sut]
|
||||
[clojure.test :refer :all]))
|
||||
|
||||
(deftest test-row-user-id->user-id
|
||||
(is (= "246364067" (sut/row-user-id->user-id "user 246364067")))
|
||||
(is (= "246364067" (sut/row-user-id->user-id "246364067"))))
|
Loading…
Reference in a new issue