mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(grfn/bbbg): Allow importing event attendees

Browse source 
Add support for importing the tsv that meetup exports into a list of
upserted attendees, and event-attendee joins.

Change-Id: I5f4ddc9fc63bcc6b0334bc3e1d3cbc4d5b99c21b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/4570
Tested-by: BuildkiteCI
Reviewed-by: grfn <grfn@gws.fyi>
Autosubmit: grfn <grfn@gws.fyi>
This commit is contained in:
Griffin Smith 2021-12-24 12:10:54 -05:00 committed by clbot
parent 92d9580df3
commit 371a444eb7
15 changed files with 242 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
users/grfn/bbbg/deps.edn
View file

@ -21,8 +21,6 @@
hiccup/hiccup {:mvn/version "1.0.5"}
garden/garden {:mvn/version "1.3.10"}
;; Utils
com.stuartsierra/component {:mvn/version "1.0.0"}
;; Logging + Observability
ch.qos.logback/logback-classic {:mvn/version "1.2.3"
@ -37,10 +35,12 @@
clj-commons/iapetos {:mvn/version "0.1.12"}
;; Utilities
com.stuartsierra/component {:mvn/version "1.0.0"}
yogthos/config {:mvn/version "1.1.8"}
clojure.java-time/clojure.java-time {:mvn/version "0.3.3"}
cheshire/cheshire {:mvn/version "5.10.1"}
org.apache.commons/commons-lang3 {:mvn/version "3.11"}
org.clojure/data.csv {:mvn/version "1.0.0"}
;; Spec
org.clojure/spec.alpha {:mvn/version "0.3.214"}

13
users/grfn/bbbg/deps.nix
View file

@ -847,6 +847,19 @@ let repos = [
paths = [ src ];
}
rec {
name = "data.csv/org.clojure";
src = fetchMavenArtifact {
inherit repos;
artifactId = "data.csv";
groupId = "org.clojure";
sha512 = "b039775a859ed27eca8f8ae74ccb6afde3ad1fe2b3cbe542240c324d60fe1237e495eb1300ee9eb4ff4ef59f01faf7aec6ef1dd6a025ee4fe556c1d91acfcf1b";
version = "1.0.0";
};
paths = [ src ];
}
rec {
name = "simpleclient_tracer_otel_agent/io.prometheus";
src = fetchMavenArtifact {

1
users/grfn/bbbg/resources/migrations/20211224161028-add-attendee-unique-meetup-id.down.sql Normal file
View file

@ -0,0 +1 @@
drop index attendee_uniq_meetup_user_id;

2
users/grfn/bbbg/resources/migrations/20211224161028-add-attendee-unique-meetup-id.up.sql Normal file
View file

@ -0,0 +1,2 @@
create unique index "attendee_uniq_meetup_user_id" on attendee (meetup_user_id);
-- ;;

4
users/grfn/bbbg/src/bbbg/attendee.clj
View file

@ -3,10 +3,8 @@
(s/def ::id uuid?)
(s/def ::meetup-name string?)
(s/def ::meetup-name (s/and string? seq))
(s/def ::discord-name (s/nilable string?))
(s/def ::meetup-user-id (s/nilable string?))
(s/def ::organizer-notes string?)

3
users/grfn/bbbg/src/bbbg/db.clj
View file

@ -355,6 +355,7 @@
(comment
(def db (:db bbbg.core/system))
(generate-migration db "add-attendee-checks")
(generate-migration db "add-attendee-unique-meetup-id")
(migrate! db)
)

16
users/grfn/bbbg/src/bbbg/db/attendee.clj
View file

@ -6,7 +6,8 @@
honeysql-postgres.helpers
[honeysql.helpers
:refer
[merge-group-by merge-join merge-left-join merge-select merge-where]]))
[merge-group-by merge-join merge-left-join merge-select merge-where]]
[bbbg.util.core :as u]))
(defn search
([q] (search {:select [:attendee.*] :from [:attendee]} q))
@ -41,6 +42,19 @@
[:not :event_attendee.attended]])
:no-shows]))))
(defn upsert-all!
[db attendees]
(db/list
db
{:insert-into :attendee
:values (map #(->> %
(db/process-key-map :attendee)
(u/map-keys keyword))
attendees)
:upsert {:on-conflict [:meetup-user-id]
:do-update-set [:meetup-name]}
:returning [:id :meetup-user-id]}))
(comment
(def db (:db bbbg.core/system))
(db/database? db)

47
users/grfn/bbbg/src/bbbg/db/attendee_check.clj
View file

@ -8,28 +8,31 @@
(defn attendees-with-last-checks
[db attendees]
(let [ids (map ::attendee/id attendees)
checks
(db/list db {:select [:attendee-check.*]
:from [:attendee-check]
:join [[{:select [:%max.attendee-check.checked-at
:attendee-check.attendee-id]
:from [:attendee-check]
:group-by [:attendee-check.attendee-id]
:where [:in :attendee-check.attendee-id ids]}
:last-check]
[:=
:attendee-check.attendee-id
:last-check.attendee-id]]})
users (u/key-by
::user/id
(db/list db {:select [:public.user.*]
:from [:public.user]
:where [:in :id (map ::user/id checks)]}))
checks (map #(assoc % :user (users (::user/id %))) checks)
attendee-id->check (u/key-by ::attendee/id checks)]
(map #(assoc % :last-check (attendee-id->check (::attendee/id %)))
attendees)))
(when (seq attendees)
(let [ids (map ::attendee/id attendees)
checks
(db/list db {:select [:attendee-check.*]
:from [:attendee-check]
:join [[{:select [:%max.attendee-check.checked-at
:attendee-check.attendee-id]
:from [:attendee-check]
:group-by [:attendee-check.attendee-id]
:where [:in :attendee-check.attendee-id ids]}
:last-check]
[:=
:attendee-check.attendee-id
:last-check.attendee-id]]})
users (if (seq checks)
(u/key-by
::user/id
(db/list db {:select [:public.user.*]
:from [:public.user]
:where [:in :id (map ::user/id checks)]}))
{})
checks (map #(assoc % :user (users (::user/id %))) checks)
attendee-id->check (u/key-by ::attendee/id checks)]
(map #(assoc % :last-check (attendee-id->check (::attendee/id %)))
attendees))))
(comment
(def db (:db bbbg.core/system))

16
users/grfn/bbbg/src/bbbg/db/event_attendee.clj Normal file
View file

@ -0,0 +1,16 @@
(ns bbbg.db.event-attendee
(:require honeysql-postgres.format
[bbbg.db :as db]
[bbbg.util.core :as u]))
(defn upsert-all!
[db attendees]
(db/execute!
db
{:insert-into :event-attendee
:values (map #(->> %
(db/process-key-map :event-attendee)
(u/map-keys keyword))
attendees)
:upsert {:on-conflict [:event-id :attendee-id]
:do-update-set [:rsvpd-attending]}}))

2
users/grfn/bbbg/src/bbbg/event_attendee.clj
View file

@ -2,3 +2,5 @@
(:require [clojure.spec.alpha :as s]))
(s/def ::attended? boolean?)
(s/def ::rsvpd-attending? boolean?)

124
users/grfn/bbbg/src/bbbg/meetup/import.clj Normal file
View file

@ -0,0 +1,124 @@
(ns bbbg.meetup.import
(:require
[bbbg.attendee :as attendee]
[bbbg.db.attendee :as db.attendee]
[bbbg.db.event-attendee :as db.event-attendee]
[bbbg.event :as event]
[bbbg.event-attendee :as event-attendee]
[bbbg.meetup-user :as meetup-user]
[bbbg.util.core :as u]
[bbbg.util.spec :as u.s]
[clojure.data.csv :as csv]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[clojure.string :as str]
[expound.alpha :as exp]))
(def spreadsheet-column->key
{"Name" :name
"User ID" :user-id
"Title" :title
"Event Host" :event-host
"RSVP" :rsvp
"Guests" :guests
"RSVPed on" :rsvped-on
"Joined Group on" :joined-group-on
"URL of Member Profile" :member-profile-url})
(defn read-attendees [f]
(with-open [reader (io/reader f)]
(let [[headers & rows] (-> reader (csv/read-csv :separator \tab))
keys (map spreadsheet-column->key headers)]
(doall
(->> rows
(map (partial zipmap keys))
(map (partial u/filter-kv (fn [k _] (some? k))))
(filter (partial some (comp seq val))))))))
;;;
(s/def ::imported-attendee
(s/keys :req [::attendee/meetup-name
::meetup-user/id]))
(def key->attendee-col
{:name ::attendee/meetup-name
:user-id ::meetup-user/id})
(defn row-user-id->user-id [row-id]
(str/replace-first row-id "user " ""))
(defn check-attendee [attendee]
()
(if (s/valid? ::imported-attendee attendee)
attendee
(throw (ex-info
(str "Invalid imported attendee\n"
(exp/expound-str ::imported-attendee attendee))
(assoc (s/explain-data ::imported-attendee attendee)
::s/failure
::s/assertion-failed)))))
(defn row->attendee [r]
(u.s/assert!
::imported-attendee
(update (u/keep-keys key->attendee-col r)
::meetup-user/id row-user-id->user-id)))
;;;
(s/def ::imported-event-attendee
(s/keys :req [::event-attendee/rsvpd-attending?
::attendee/id
::event/id]))
(def key->event-attendee-col
{:rsvp ::event-attendee/rsvpd-attending?})
(defn row->event-attendee
[{event-id ::event/id :keys [meetup-id->attendee-id]} r]
(let [attendee-id (-> r :user-id row-user-id->user-id meetup-id->attendee-id)]
(u.s/assert!
::imported-event-attendee
(-> (u/keep-keys key->event-attendee-col r)
(update ::event-attendee/rsvpd-attending?
(partial = "Yes"))
(assoc ::event/id event-id
::attendee/id attendee-id)))))
;;;
(defn import-data! [db event-id f]
(let [rows (read-attendees f)
attendees (db.attendee/upsert-all! db (map row->attendee rows))
meetup-id->attendee-id (into {}
(map (juxt ::meetup-user/id ::attendee/id))
attendees)]
(db.event-attendee/upsert-all!
db
(map (partial row->event-attendee
{::event/id event-id
:meetup-id->attendee-id meetup-id->attendee-id})
rows))))
;;; Spreadsheet columns:
;;;
;;; Name
;;; User ID
;;; Title
;;; Event Host
;;; RSVP
;;; Guests
;;; RSVPed on
;;; Joined Group on
;;; URL of Member Profile
;;; Have you been to one of our events before? Note, attendance at all events will require proof of vaccination until further notice.
(comment
(def -filename- "/home/grfn/code/depot/users/grfn/bbbg/sample-data.tsv")
(def event-id #uuid "09f8fed6-7480-451b-89a2-bb4edaeae657")
(read-attendees -filename-)
(import-data! (:db bbbg.core/system) event-id -filename-)
)

8
users/grfn/bbbg/src/bbbg/meetup_user.clj Normal file
View file

@ -0,0 +1,8 @@
(ns bbbg.meetup-user
(:require [clojure.spec.alpha :as s]))
(s/def ::id
(s/nilable
(s/and string?
seq
#(re-matches #"\d+" %))))

8
users/grfn/bbbg/src/bbbg/util/core.clj
View file

@ -52,6 +52,14 @@
([f] (map-kv f identity))
([f m] (map-kv f identity m)))
(defn keep-keys
"Map f over the keys of m, keeping only those entries for which f does not
return nil. Preserves metadata on the incoming map. The one-argument form
returns a transducer that yields map-entries."
([f] (keep (fn [[k v]] (when-let [k' (f k)]
(first {k' v})))))
([f m] (into (empty m) (keep-keys f) m)))
(defn map-vals
"Map f over the values of m. Preserves metadata on the incoming map. The
one-argument form returns a transducer that yields map-entries."

16
users/grfn/bbbg/src/bbbg/util/spec.clj Normal file
View file

@ -0,0 +1,16 @@
(ns bbbg.util.spec
(:require [expound.alpha :as exp]
[clojure.spec.alpha :as s]))
(defn assert!
([spec s] (assert! "Spec assertion failed" spec s))
([message spec x]
(if (s/valid? spec x)
x
(throw (ex-info
(str message
"\n"
(exp/expound-str spec x))
(assoc (s/explain-data spec x)
::s/failure
::s/assertion-failed))))))

7
users/grfn/bbbg/test/bbbg/meetup/import_test.clj Normal file
View file

@ -0,0 +1,7 @@
(ns bbbg.meetup.import-test
(:require [bbbg.meetup.import :as sut]
[clojure.test :refer :all]))
(deftest test-row-user-id->user-id
(is (= "246364067" (sut/row-user-id->user-id "user 246364067")))
(is (= "246364067" (sut/row-user-id->user-id "246364067"))))