Move monzo_ynab into //tools

Optimizing is difficult: I like flat hierarchies because I don't like
directory-hopping, but I also would like a cleaner root for my mono-repo. Bombs
away!

Well it's that time again, folks: spring cleaning!

Here I am musing about a few things that bother me:
- Should I use kebab-case or snake_case?
- It feels ~confusing to have //tools and //utils. What a //projects? Isn't
  everything a project? *sigh*
This commit is contained in:
William Carroll 2020-07-20 10:15:47 +01:00
parent 4ed3254709
commit 5add8ddc13
16 changed files with 3 additions and 3 deletions

7
tools/monzo_ynab/.envrc Normal file
View file

@ -0,0 +1,7 @@
source_up
export monzo_client_id="$(pass show finance/monzo/client-id)"
export monzo_client_secret="$(pass show finance/monzo/client-secret)"
export ynab_personal_access_token="$(pass show finance/youneedabudget.com/personal-access-token)"
export ynab_account_id="$(pass show finance/youneedabudget.com/personal-access-token)"
export ynab_budget_id="$(pass show finance/youneedabudget.com/budget-id)"
export store_path="$(pwd)"

3
tools/monzo_ynab/.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
/ynab/fixture.json
/monzo/fixture.json
/kv.json

View file

@ -0,0 +1,41 @@
# monzo_ynab
Exporting Monzo transactions to my YouNeedABudget.com (i.e. YNAB) account. YNAB
unfortunately doesn't currently offer an Monzo integration. As a workaround and
a practical excuse to learn Go, I decided to write one myself.
This job is going to run N times per 24 hours. Monzo offers webhooks for
reacting to certain types of events. I don't expect I'll need realtime data for
my YNAB integration. That may change, however, so it's worth noting.
## Installation
Like many other packages in this repository, `monzo_ynab` is packaged using
Nix. To install and use, you have two options:
You can install using `nix-build` and then run the resulting
`./result/bin/monzo_ynab`.
```shell
> nix-build . && ./result/bin/monzo_ynab
```
Or you can install using `nix-env` if you'd like to create the `monzo_ynab`
symlink.
```shell
> nix-env -f ~/briefcase/monzo_ynab -i
```
## Deployment
While this project is currently not deployed, my plan is to host it on Google
Cloud and run it as a Cloud Run application. What I don't yet know is whether or
not this is feasible or a good idea. One complication that I foresee is that the
OAuth 2.0 login flow requires a web browser until the access token and refresh
tokens are acquired. I'm unsure how to workaround this at the moment.
For more information about the general packaging and deployment strategies I'm
currently using, refer to the [deployments][deploy] writeup.
[deploy]: ../deploy/README.md

101
tools/monzo_ynab/auth.go Normal file
View file

@ -0,0 +1,101 @@
package auth
////////////////////////////////////////////////////////////////////////////////
// Dependencies
////////////////////////////////////////////////////////////////////////////////
import (
"encoding/json"
"fmt"
"log"
"net/http"
"net/url"
"os"
"os/exec"
"utils"
)
////////////////////////////////////////////////////////////////////////////////
// Constants
////////////////////////////////////////////////////////////////////////////////
var (
BROWSER = os.Getenv("BROWSER")
REDIRECT_URI = "http://localhost:8080/authorization-code"
)
////////////////////////////////////////////////////////////////////////////////
// Types
////////////////////////////////////////////////////////////////////////////////
// This is the response returned from Monzo when we exchange our authorization
// code for an access token. While Monzo returns additional fields, I'm only
// interested in AccessToken and RefreshToken.
type accessTokenResponse struct {
AccessToken string `json:"access_token"`
RefreshToken string `json:"refresh_token"`
ExpiresIn int `json:"expires_in"`
}
type Tokens struct {
AccessToken string
RefreshToken string
ExpiresIn int
}
////////////////////////////////////////////////////////////////////////////////
// Functions
////////////////////////////////////////////////////////////////////////////////
// Returns the access token and refresh tokens for the Monzo API.
func GetTokensFromAuthCode(authCode string, clientID string, clientSecret string) *Tokens {
res, err := http.PostForm("https://api.monzo.com/oauth2/token", url.Values{
"grant_type": {"authorization_code"},
"client_id": {clientID},
"client_secret": {clientSecret},
"redirect_uri": {REDIRECT_URI},
"code": {authCode},
})
utils.FailOn(err)
defer res.Body.Close()
payload := &accessTokenResponse{}
json.NewDecoder(res.Body).Decode(payload)
return &Tokens{payload.AccessToken, payload.RefreshToken, payload.ExpiresIn}
}
// Open a web browser to allow the user to authorize this application. Return
// the authorization code sent from Monzo.
func GetAuthCode(clientID string) string {
// TODO(wpcarro): Consider generating a random string for the state when the
// application starts instead of hardcoding it here.
state := "xyz123"
url := fmt.Sprintf(
"https://auth.monzo.com/?client_id=%s&redirect_uri=%s&response_type=code&state=%s",
clientID, REDIRECT_URI, state)
exec.Command(BROWSER, url).Start()
authCode := make(chan string)
go func() {
log.Fatal(http.ListenAndServe(":8080",
http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
// 1. Get authorization code from Monzo.
if req.URL.Path == "/authorization-code" {
params := req.URL.Query()
reqState := params["state"][0]
code := params["code"][0]
if reqState != state {
log.Fatalf("Value for state returned by Monzo does not equal our state. %s != %s", reqState, state)
}
authCode <- code
fmt.Fprintf(w, "Authorized!")
} else {
log.Printf("Unhandled request: %v\n", *req)
}
})))
}()
result := <-authCode
return result
}

View file

@ -0,0 +1,3 @@
let
briefcase = import <briefcase> {};
in briefcase.utils.nixBufferFromShell ./shell.nix

12
tools/monzo_ynab/job.nix Normal file
View file

@ -0,0 +1,12 @@
{ depot, briefcase, ... }:
depot.buildGo.program {
name = "job";
srcs = [
./main.go
];
deps = with briefcase.gopkgs; [
kv
utils
];
}

43
tools/monzo_ynab/main.go Normal file
View file

@ -0,0 +1,43 @@
// Exporting Monzo transactions to my YouNeedABudget.com (i.e. YNAB)
// account. YNAB unfortunately doesn't currently offer an Monzo integration. As
// a workaround and a practical excuse to learn Go, I decided to write one
// myself.
//
// This job is going to run N times per 24 hours. Monzo offers webhooks for
// reacting to certain types of events. I don't expect I'll need realtime data
// for my YNAB integration. That may change, however, so it's worth noting.
package main
import (
"fmt"
)
var (
ynabAccountID = os.Getenv("ynab_account_id")
)
////////////////////////////////////////////////////////////////////////////////
// Business Logic
////////////////////////////////////////////////////////////////////////////////
// Convert a Monzo transaction struct, `tx`, into a YNAB transaction struct.
func toYnab(tx monzoSerde.Transaction) ynabSerde.Transaction {
return ynabSerde.Transaction{
Id: tx.Id,
Date: tx.Created,
Amount: tx.Amount,
Memo: tx.Notes,
AccountId: ynabAccountID,
}
}
func main() {
txs := monzo.TransactionsLast24Hours()
var ynabTxs []ynabSerde.Transaction{}
for tx := range txs {
append(ynabTxs, toYnab(tx))
}
ynab.PostTransactions(ynabTxs)
os.Exit(0)
}

View file

@ -0,0 +1,52 @@
package monzoClient
import (
"fmt"
"log"
"monzoSerde"
"net/http"
"net/url"
"strings"
"time"
"tokens"
"utils"
)
const (
accountID = "pizza"
)
type Client struct{}
// Ensure that the token server is running and return a new instance of a Client
// struct.
func Create() *Client {
tokens.StartServer()
time.Sleep(time.Second * 1)
return &Client{}
}
// Returns a slice of transactions from the last 24 hours.
func (c *Client) Transactions24Hours() []monzoSerde.Transaction {
token := tokens.AccessToken()
form := url.Values{"account_id": {accountID}}
client := http.Client{}
req, _ := http.NewRequest("POST", "https://api.monzo.com/transactions",
strings.NewReader(form.Encode()))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
req.Header.Add("User-Agent", "monzo-ynab")
res, err := client.Do(req)
utils.DebugRequest(req)
utils.DebugResponse(res)
if err != nil {
utils.DebugRequest(req)
utils.DebugResponse(res)
log.Fatal(err)
}
defer res.Body.Close()
return []monzoSerde.Transaction{}
}

View file

@ -0,0 +1,82 @@
// This package hosts the serialization and deserialization logic for all of the
// data types with which our application interacts from the Monzo API.
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"time"
)
type TxMetadata struct {
FasterPayment string `json:"faster_payment"`
FpsPaymentId string `json:"fps_payment_id"`
Insertion string `json:"insertion"`
Notes string `json:"notes"`
Trn string `json:"trn"`
}
type TxCounterparty struct {
AccountNumber string `json:"account_number"`
Name string `json:"name"`
SortCode string `json:"sort_code"`
UserId string `json:"user_id"`
}
type Transaction struct {
Id string `json:"id"`
Created time.Time `json:"created"`
Description string `json:"description"`
Amount int `json:"amount"`
Currency string `json:"currency"`
Notes string `json:"notes"`
Metadata TxMetadata
AccountBalance int `json:"account_balance"`
International interface{} `json:"international"`
Category string `json:"category"`
IsLoad bool `json:"is_load"`
Settled time.Time `json:"settled"`
LocalAmount int `json:"local_amount"`
LocalCurrency string `json:"local_currency"`
Updated time.Time `json:"updated"`
AccountId string `json:"account_id"`
UserId string `json:"user_id"`
Counterparty TxCounterparty `json:"counterparty"`
Scheme string `json:"scheme"`
DedupeId string `json:"dedupe_id"`
Originator bool `json:"originator"`
IncludeInSpending bool `json:"include_in_spending"`
CanBeExcludedFromBreakdown bool `json:"can_be_excluded_from_breakdown"`
CanBeMadeSubscription bool `json:"can_be_made_subscription"`
CanSplitTheBill bool `json:"can_split_the_bill"`
CanAddToTab bool `json:"can_add_to_tab"`
AmountIsPending bool `json:"amount_is_pending"`
// Fees interface{} `json:"fees"`
// Merchant interface `json:"merchant"`
// Labels interface{} `json:"labels"`
// Attachments interface{} `json:"attachments"`
// Categories interface{} `json:"categories"`
}
// Attempts to encode a Monzo transaction struct into a string.
func serializeTx(tx *Transaction) (string, error) {
x, err := json.Marshal(tx)
return string(x), err
}
// Attempts to parse a string encoding a transaction presumably sent from a
// Monzo server.
func deserializeTx(x string) (*Transaction, error) {
target := &Transaction{}
err := json.Unmarshal([]byte(x), target)
return target, err
}
func main() {
b, _ := ioutil.ReadFile("./fixture.json")
tx := string(b)
target, _ := deserializeTx(tx)
out, _ := serializeTx(target)
fmt.Println(out)
}

View file

@ -0,0 +1,80 @@
################################################################################
# YNAB
################################################################################
:ynab = https://api.youneedabudget.com/v1
:ynab-access-token := (getenv "ynab_personal_access_token")
:ynab-budget-id := (getenv "ynab_budget_id")
:ynab-account-id := (getenv "ynab_account_id")
# Test
GET :ynab/budgets
Authorization: Bearer :ynab-access-token
# List transactions
GET :ynab/budgets/:ynab-budget-id/transactions
Authorization: Bearer :ynab-access-token
# Post transactions
POST :ynab/budgets/:ynab-budget-id/transactions
Authorization: Bearer :ynab-access-token
Content-Type: application/json
{
"transactions": [
{
"account_id": ":ynab-account-id",
"date": "2019-12-30",
"amount": 10000,
"payee_name": "Richard Stallman",
"memo": "Not so free software after all...",
"cleared": "cleared",
"approved": true,
"flag_color": "red",
"import_id": "xyz-123"
}
]
}
################################################################################
# Monzo
################################################################################
:monzo = https://api.monzo.com
:monzo-access-token := (getenv "monzo_cached_access_token")
:monzo-refresh-token := (getenv "monzo_cached_refresh_token")
:monzo-client-id := (getenv "monzo_client_id")
:monzo-client-secret := (getenv "monzo_client_secret")
:monzo-account-id := (getenv "monzo_account_id")
# List transactions
GET :monzo/transactions
Authorization: Bearer :monzo-access-token
account_id==:monzo-account-id
# Refresh access token
# According from the docs, the access token expires in 6 hours.
POST :monzo/oauth2/token
Content-Type: application/x-www-form-urlencoded
Authorization: Bearer :monzo-access-token
grant_type=refresh_token&client_id=:monzo-client-id&client_secret=:monzo-client-secret&refresh_token=:monzo-refresh-token
################################################################################
# Tokens server
################################################################################
:tokens = http://localhost:4242
# Get tokens
GET :tokens/tokens
# Get application state for debugging purposes
GET :tokens/state
# Force refresh tokens
POST :tokens/refresh-tokens
# Set tokens
POST :tokens/set-tokens
Content-Type: application/json
{
"access_token": "access-token",
"refresh_token": "refresh-token",
"expires_in": 120
}

View file

@ -0,0 +1,9 @@
let
pkgs = import <nixpkgs> {};
in pkgs.mkShell {
buildInputs = [
pkgs.go
pkgs.goimports
pkgs.godef
];
}

283
tools/monzo_ynab/tokens.go Normal file
View file

@ -0,0 +1,283 @@
// Creating a Tokens server to manage my access and refresh tokens. Keeping this
// as a separate server allows me to develop and use the access tokens without
// going through client authorization.
package main
////////////////////////////////////////////////////////////////////////////////
// Dependencies
////////////////////////////////////////////////////////////////////////////////
import (
"auth"
"encoding/json"
"fmt"
"io"
"kv"
"log"
"net/http"
"net/url"
"os"
"os/signal"
"syscall"
"time"
"utils"
)
////////////////////////////////////////////////////////////////////////////////
// Types
////////////////////////////////////////////////////////////////////////////////
// This is the response from Monzo's API after we request an access token
// refresh.
type refreshTokenResponse struct {
AccessToken string `json:"access_token"`
RefreshToken string `json:"refresh_token"`
ClientId string `json:"client_id"`
ExpiresIn int `json:"expires_in"`
}
// This is the shape of the request from clients wishing to set state of the
// server.
type setTokensRequest struct {
AccessToken string `json:"access_token"`
RefreshToken string `json:"refresh_token"`
ExpiresIn int `json:"expires_in"`
}
// This is our application state.
type state struct {
accessToken string `json:"access_token"`
refreshToken string `json:"refresh_token"`
}
type readMsg struct {
sender chan state
}
type writeMsg struct {
state state
sender chan bool
}
type channels struct {
reads chan readMsg
writes chan writeMsg
}
////////////////////////////////////////////////////////////////////////////////
// Top-level Definitions
////////////////////////////////////////////////////////////////////////////////
var chans = &channels{
reads: make(chan readMsg),
writes: make(chan writeMsg),
}
var (
monzoClientId = os.Getenv("monzo_client_id")
monzoClientSecret = os.Getenv("monzo_client_secret")
storePath = os.Getenv("store_path")
)
////////////////////////////////////////////////////////////////////////////////
// Utils
////////////////////////////////////////////////////////////////////////////////
// Print the access and refresh tokens for debugging.
func logTokens(access string, refresh string) {
log.Printf("Access: %s\n", access)
log.Printf("Refresh: %s\n", refresh)
}
func (state *state) String() string {
return fmt.Sprintf("state{\n\taccessToken: \"%s\",\n\trefreshToken: \"%s\"\n}\n", state.accessToken, state.refreshToken)
}
// Schedule a token refresh for `expiresIn` seconds using the provided
// `refreshToken`. This will update the application state with the access token
// and schedule an additional token refresh for the newly acquired tokens.
func scheduleTokenRefresh(expiresIn int, refreshToken string) {
duration := time.Second * time.Duration(expiresIn)
timestamp := time.Now().Local().Add(duration)
// TODO(wpcarro): Consider adding a more human readable version that will
// log the number of hours, minutes, etc. until the next refresh.
log.Printf("Scheduling token refresh for %v\n", timestamp)
time.Sleep(duration)
log.Println("Refreshing tokens now...")
accessToken, refreshToken := refreshTokens(refreshToken)
log.Println("Successfully refreshed tokens.")
logTokens(accessToken, refreshToken)
setState(accessToken, refreshToken)
}
// Exchange existing credentials for a new access token and `refreshToken`. Also
// schedule the next refresh. This function returns the newly acquired access
// token and refresh token.
func refreshTokens(refreshToken string) (string, string) {
// TODO(wpcarro): Support retries with exponential backoff.
res, err := http.PostForm("https://api.monzo.com/oauth2/token", url.Values{
"grant_type": {"refresh_token"},
"client_id": {monzoClientId},
"client_secret": {monzoClientSecret},
"refresh_token": {refreshToken},
})
if res.StatusCode != http.StatusOK {
// TODO(wpcarro): Considering panicking here.
utils.DebugResponse(res)
}
if err != nil {
utils.DebugResponse(res)
log.Fatal("The request to Monzo to refresh our access token failed.", err)
}
defer res.Body.Close()
payload := &refreshTokenResponse{}
err = json.NewDecoder(res.Body).Decode(payload)
if err != nil {
log.Fatal("Could not decode the JSON response from Monzo.", err)
}
go scheduleTokenRefresh(payload.ExpiresIn, payload.RefreshToken)
// Interestingly, JSON decoding into the refreshTokenResponse can success
// even if the decoder doesn't populate any of the fields in the
// refreshTokenResponse struct. From what I read, it isn't possible to make
// these fields as required using an annotation, so this guard must suffice
// for now.
if payload.AccessToken == "" || payload.RefreshToken == "" {
log.Fatal("JSON parsed correctly but failed to populate token fields.")
}
return payload.AccessToken, payload.RefreshToken
}
func persistTokens(access string, refresh string) {
log.Println("Persisting tokens...")
kv.Set(storePath, "monzoAccessToken", access)
kv.Set(storePath, "monzoRefreshToken", refresh)
log.Println("Successfully persisted tokens.")
}
// Listen for SIGINT and SIGTERM signals. When received, persist the access and
// refresh tokens and shutdown the server.
func handleInterrupts() {
// Gracefully handle interruptions.
sigs := make(chan os.Signal, 1)
done := make(chan bool)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig := <-sigs
log.Printf("Received signal to shutdown. %v\n", sig)
state := getState()
persistTokens(state.accessToken, state.refreshToken)
done <- true
}()
<-done
log.Println("Exiting...")
os.Exit(0)
}
// Set `accessToken` and `refreshToken` on application state.
func setState(accessToken string, refreshToken string) {
msg := writeMsg{state{accessToken, refreshToken}, make(chan bool)}
chans.writes <- msg
<-msg.sender
}
// Return our application state.
func getState() state {
msg := readMsg{make(chan state)}
chans.reads <- msg
return <-msg.sender
}
////////////////////////////////////////////////////////////////////////////////
// Main
////////////////////////////////////////////////////////////////////////////////
func main() {
// Manage application state.
go func() {
state := &state{}
for {
select {
case msg := <-chans.reads:
log.Println("Reading from state...")
log.Println(state)
msg.sender <- *state
case msg := <-chans.writes:
log.Println("Writing to state.")
log.Printf("Old: %s\n", state)
*state = msg.state
log.Printf("New: %s\n", state)
// As an attempt to maintain consistency between application
// state and persisted state, everytime we write to the
// application state, we will write to the store.
persistTokens(state.accessToken, state.refreshToken)
msg.sender <- true
}
}
}()
// Retrieve cached tokens from store.
accessToken := fmt.Sprintf("%v", kv.Get(storePath, "monzoAccessToken"))
refreshToken := fmt.Sprintf("%v", kv.Get(storePath, "monzoRefreshToken"))
log.Println("Attempting to retrieve cached credentials...")
logTokens(accessToken, refreshToken)
if accessToken == "" || refreshToken == "" {
log.Println("Cached credentials are absent. Authorizing client...")
authCode := auth.GetAuthCode(monzoClientId)
tokens := auth.GetTokensFromAuthCode(authCode, monzoClientId, monzoClientSecret)
setState(tokens.AccessToken, tokens.RefreshToken)
go scheduleTokenRefresh(tokens.ExpiresIn, tokens.RefreshToken)
} else {
setState(accessToken, refreshToken)
// If we have tokens, they may be expiring soon. We don't know because
// we aren't storing the expiration timestamp in the state or in the
// store. Until we have that information, and to be safe, let's refresh
// the tokens.
go scheduleTokenRefresh(0, refreshToken)
}
// Gracefully handle shutdowns.
go handleInterrupts()
// Listen to inbound requests.
fmt.Println("Listening on http://localhost:4242 ...")
log.Fatal(http.ListenAndServe(":4242",
http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if req.URL.Path == "/refresh-tokens" && req.Method == "POST" {
state := getState()
go scheduleTokenRefresh(0, state.refreshToken)
fmt.Fprintf(w, "Done.")
} else if req.URL.Path == "/set-tokens" && req.Method == "POST" {
// Parse
payload := &setTokensRequest{}
err := json.NewDecoder(req.Body).Decode(payload)
if err != nil {
log.Fatal("Could not decode the user's JSON request.", err)
}
// Update application state
setState(payload.AccessToken, payload.RefreshToken)
// Refresh tokens
go scheduleTokenRefresh(payload.ExpiresIn, payload.RefreshToken)
// Ack
fmt.Fprintf(w, "Done.")
} else if req.URL.Path == "/state" && req.Method == "GET" {
// TODO(wpcarro): Ensure that this returns serialized state.
w.Header().Set("Content-type", "application/json")
state := getState()
payload, _ := json.Marshal(state)
io.WriteString(w, string(payload))
} else {
log.Printf("Unhandled request: %v\n", *req)
}
})))
}

View file

@ -0,0 +1,23 @@
{ depot, briefcase, ... }:
let
auth = depot.buildGo.package {
name = "auth";
srcs = [
./auth.go
];
deps = with briefcase.gopkgs; [
utils
];
};
in depot.buildGo.program {
name = "token-server";
srcs = [
./tokens.go
];
deps = with briefcase.gopkgs; [
kv
utils
auth
];
}

View file

@ -0,0 +1,24 @@
package client
import (
"serde"
)
// See requests.txt for more details.
func PostTransactions(accountID string, txs []serde.Transaction{}) error {
return map[string]string{
"transactions": [
{
"account_id": accountID,
"date": "2019-12-30",
"amount": 10000,
"payee_name": "Richard Stallman",
"memo": "Not so free software after all...",
"cleared": "cleared",
"approved": true,
"flag_color": "red",
"import_id": "xyz-123"
}
]
}
}

View file

@ -0,0 +1,52 @@
// This package hosts the serialization and deserialization logic for all of the
// data types with which our application interacts from the YNAB API.
package main
import (
"encoding/json"
"fmt"
"time"
)
type Transaction struct {
Id string `json:"id"`
Date time.Time `json:"date"`
Amount int `json:"amount"`
Memo string `json:"memo"`
Cleared string `json:"cleared"`
Approved bool `json:"approved"`
FlagColor string `json:"flag_color"`
AccountId string `json:"account_id"`
AccountName string `json:"account_name"`
PayeeId string `json:"payeed_id"`
PayeeName string `json:"payee_name"`
CategoryId string `json:"category_id"`
CategoryName string `json:"category_name"`
Deleted bool `json:"deleted"`
// TransferAccountId interface{} `json:"transfer_account_id"`
// TransferTransactionId interface{} `json:"transfer_transaction_id"`
// MatchedTransactionId interface{} `json:"matched_transaction_id"`
// ImportId interface{} `json:"import_id"`
// Subtransactions interface{} `json:"subtransactions"`
}
// Attempts to encode a YNAB transaction into a string.
func serializeTx(tx *Transaction) (string, error) {
x, err := json.Marshal(tx)
return string(x), err
}
// Attempts to parse a string encoding a transaction presumably sent from a
// YNAB server.
func deserializeTx(x string) (*Transaction, error) {
target := &Transaction{}
err := json.Unmarshal([]byte(x), target)
return target, err
}
func main() {
target, _ := deserializeTx(tx)
out, _ := serializeTx(target)
fmt.Println(out)
fmt.Println(ynabOut)
}