mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ebooks): Add tvl-ebooks code

Browse source 
Change-Id: If519e789a91fbf427373daa383c6ae00ba5e0b6c
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2007
Tested-by: BuildkiteCI
Reviewed-by: tazjin <mail@tazj.in>
This commit is contained in:
Ben Cartwright-Cox 2020-09-28 00:22:05 +01:00 committed by ben
parent cf878224ab
commit a8508373ee
8 changed files with 687 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
fun/tvl-ebooks/OWNERS Normal file
View file

@ -0,0 +1,3 @@
inherited: true
owners:
- ben

170
fun/tvl-ebooks/ebook-client/main.go Normal file
View file

@ -0,0 +1,170 @@
package main
import (
"crypto/tls"
"encoding/json"
"flag"
"fmt"
"log"
"net"
"sync"
"time"
"github.com/go-redis/redis"
"gopkg.in/irc.v3"
)
var messageBeat chan bool
var firstMessage chan bool
var client *irc.Client
var safeLock sync.Mutex
func main() {
nick := flag.String("nick", "NONE", "the ircnick you want")
from := flag.String("ip", "[::1]", "src address")
flag.Parse()
localAddrDialier := &net.Dialer{
LocalAddr: &net.TCPAddr{
IP: net.ParseIP(*from),
Port: 0,
},
}
conn, err := tls.DialWithDialer(localAddrDialier, "tcp", "chat.freenode.net:6697", &tls.Config{})
if err != nil {
log.Fatalln(err)
}
messageBeat = make(chan bool)
firstMessage = make(chan bool, 10)
go ircKeepalive()
redisc := redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("127.0.0.1:%d", 6379),
Password: "", // no password set
DB: 0, // use default DB
})
go func() {
for {
time.Sleep(time.Second)
r := redisc.Ping()
if r.Err() != nil {
redisc = redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("127.0.0.1:%d", 6379),
Password: "", // no password set
DB: 0, // use default DB
})
}
redisc.Set(fmt.Sprintf("alive-%s", *nick), "yes", time.Second*5)
}
}()
if *nick == "NONE" {
log.Fatalf("You must set a nick")
}
go func() {
<-firstMessage
for {
psub := redisc.Subscribe(fmt.Sprintf("irc-%s", *nick))
for {
msg, err := psub.ReceiveMessage()
if err != nil {
break
}
client.WriteMessage(&irc.Message{
Command: "PRIVMSG",
Params: []string{
"##tvl-ebooks",
msg.Payload,
},
})
}
time.Sleep(time.Second * 10)
}
}()
go func() {
<-firstMessage
for {
psub := redisc.Subscribe(fmt.Sprintf("raw-irc-%s", *nick))
for {
msg, err := psub.ReceiveMessage()
if err != nil {
break
}
im := irc.Message{}
err = json.Unmarshal([]byte(msg.Payload), &im)
if err == nil {
client.WriteMessage(&im)
}
}
time.Sleep(time.Second * 10)
}
}()
seenMsgBefore := false
config := irc.ClientConfig{
Nick: *nick,
User: *nick,
Name: fmt.Sprintf("%s Ebooks", *nick),
Handler: irc.HandlerFunc(func(c *irc.Client, m *irc.Message) {
b, _ := json.Marshal(m)
log.Printf("%#v", string(b))
messageBeat <- true
if !seenMsgBefore {
firstMessage <- true
seenMsgBefore = true
}
res := redisc.Publish("ebook", string(b))
if res.Err() != nil {
log.Printf("Publish error! %#v", err)
}
if m.Command == "001" {
// 001 is a welcome event, so we join channels there
c.Write("JOIN ##tvl-ebooks")
}
// else if m.Command == "PRIVMSG" && c.FromChannel(m) {
// // // Create a handler on all messages.
// // c.WriteMessage(&irc.Message{
// // Command: "PRIVMSG",
// // Params: []string{
// // m.Params[0],
// // m.Trailing(),
// // },
// // })
// }
}),
}
// Create the client
client = irc.NewClient(conn, config)
err = client.Run()
if err != nil {
log.Fatalln(err)
}
}
func ircKeepalive() {
tt := time.NewTimer(time.Second)
lastPing := time.Now()
for {
select {
case <-tt.C:
if time.Since(lastPing) > time.Minute*5 {
log.Fatalf("It's been too long since the last IRC message, blowing up")
}
break
case <-messageBeat:
lastPing = time.Now()
}
}
}

8
fun/tvl-ebooks/go.mod Normal file
View file

@ -0,0 +1,8 @@
module github.com/benjojo/tvl-ebooks
go 1.14
require (
github.com/go-redis/redis v6.15.9+incompatible
gopkg.in/irc.v3 v3.1.3
)

11
fun/tvl-ebooks/go.sum Normal file
View file

@ -0,0 +1,11 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/irc.v3 v3.1.3 h1:yeTiJ365882L8h4AnBKYfesD92y5R5ZhGiylu9DfcPY=
gopkg.in/irc.v3 v3.1.3/go.mod h1:shO2gz8+PVeS+4E6GAny88Z0YVVQSxQghdrMVGQsR9s=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

99
fun/tvl-ebooks/irc-ingest/main.go Normal file
View file

@ -0,0 +1,99 @@
package main
import (
"crypto/tls"
"encoding/json"
"fmt"
"log"
"os"
"time"
"github.com/go-redis/redis"
"gopkg.in/irc.v3"
)
var messageBeat chan bool
func main() {
conn, err := tls.Dial("tcp", "bnc.irccloud.com:6697", nil)
if err != nil {
log.Fatalln(err)
}
messageBeat = make(chan bool)
go ircKeepalive()
redisc := redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("127.0.0.1:%d", 6379),
Password: "", // no password set
DB: 0, // use default DB
})
go func() {
for {
time.Sleep(time.Second)
r := redisc.Ping()
if r.Err() != nil {
redisc = redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("127.0.0.1:%d", 6379),
Password: "", // no password set
DB: 0, // use default DB
})
}
}
}()
config := irc.ClientConfig{
Nick: "Benjojo",
Pass: os.Getenv("IRCCLOUD"),
User: "Benjojo",
Name: "Ben Cox",
Handler: irc.HandlerFunc(func(c *irc.Client, m *irc.Message) {
b, _ := json.Marshal(m)
// log.Printf("%#v", string(b))
messageBeat <- true
res := redisc.Publish("irccloud", string(b))
if res.Err() != nil {
log.Printf("Publish error! %#v", err)
}
// if m.Command == "001" {
// // 001 is a welcome event, so we join channels there
// // c.Write("JOIN #bot-test-chan")
// } else if m.Command == "PRIVMSG" && c.FromChannel(m) {
// // // Create a handler on all messages.
// // c.WriteMessage(&irc.Message{
// // Command: "PRIVMSG",
// // Params: []string{
// // m.Params[0],
// // m.Trailing(),
// // },
// // })
// }
}),
}
// Create the client
client := irc.NewClient(conn, config)
err = client.Run()
if err != nil {
log.Fatalln(err)
}
}
func ircKeepalive() {
tt := time.NewTimer(time.Second)
lastPing := time.Now()
for {
select {
case <-tt.C:
if time.Since(lastPing) > time.Minute*5 {
log.Fatalf("It's been too long since the last IRC message, blowing up")
}
break
case <-messageBeat:
lastPing = time.Now()
}
}
}

26
fun/tvl-ebooks/make-v6/main.go Normal file
View file

@ -0,0 +1,26 @@
package main
import (
"crypto/rand"
"log"
"net"
)
func main() {
// 2a0c:2f07:29:9999:6564:5298:8413:4652
ip := net.ParseIP("2a0c:2f07:29::")
rand.Read(ip[6:])
if ip[7] > 0xaa {
ip[4] = 0x03
ip[5] = 0x84
if ip[7] > 0xdd {
ip[4] = 0x08
ip[5] = 0x64
}
}
log.Printf("%s\n", ip)
//
}

196
fun/tvl-ebooks/mkov-engine/main.go Normal file
View file

@ -0,0 +1,196 @@
package main
import (
"encoding/json"
"fmt"
"log"
"math/rand"
"strconv"
"strings"
"time"
"github.com/go-redis/redis"
)
type incomingIRC struct {
Command string `json:"Command"`
Host string `json:"Host"`
Name string `json:"Name"`
Params []string `json:"Params"`
User string `json:"User"`
}
var supressionUsernames map[string]bool
func main() {
redisc := redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("127.0.0.1:%d", 6379),
Password: "", // no password set
DB: 0, // use default DB
})
fireaway := make(chan incomingIRC, 10)
supressionUsernames = make(map[string]bool)
go func() {
for {
irccloudFeed := redisc.Subscribe("irccloud")
for {
msg, err := irccloudFeed.ReceiveMessage()
if err != nil {
break
}
imsg := incomingIRC{}
err = json.Unmarshal([]byte(msg.Payload), &imsg)
if err != nil {
log.Printf("Json decoding error from irccloud feed %s", err)
continue
}
if imsg.Command == "PRIVMSG" {
if len(imsg.Params) == 2 {
if imsg.Params[0] == "##tvl" || imsg.Params[0] == "##tvlbot" {
fireaway <- imsg
}
}
}
}
time.Sleep(time.Second)
}
}()
for msg := range fireaway {
// Learn
learnFromMessage(msg, redisc)
msg2 := generateMesasge(msg, redisc)
// Check if we have a active log in for that user
ttl := redisc.TTL("alive-" + msg.Name + "-eb")
ttld, err := ttl.Result()
if err == nil {
redisc.Publish("irc-"+msg.Name+"-eb", msg2)
if ttld == 0 || ttld.Seconds() == -2 {
redisc.Publish("irc-tvlebooks-eb", "<"+fmt.Sprintf("%s.%s", string(msg.Name[0]), msg.Name[1:])+"-eb> "+msg2)
}
} else {
redisc.Publish("irc-tvlebooks-eb", "<"+fmt.Sprintf("%s.%s", string(msg.Name[0]), msg.Name[1:])+"-eb> "+msg2)
}
}
}
func generateMesasge(msg incomingIRC, redisc *redis.Client) string {
text := msg.Params[1]
username := msg.Name
supressionUsernames[strings.ToLower(username)] = true
supressionUsernames[strings.ToLower(username)+":"] = true
text = strings.ToLower(text)
text = strings.Replace(text, ",", "", -1)
text = strings.Replace(text, ",", "", -1)
text = strings.Replace(text, ".", "", -1)
text = strings.Replace(text, "!", "", -1)
text = strings.Replace(text, "?", "", -1)
words := strings.Split(text, " ")
lastWord := propwords(username, words[0], redisc)
if supressionUsernames[words[0]] {
if len(words[0]) < 2 {
words[0] = "vee"
}
words[0] = fmt.Sprintf("%s.%s", string(words[0][0]), words[0][1:])
}
if lastWord == "_END_" {
return words[0]
}
outputMsg := words[0] + " " + lastWord + " "
for {
lastWord = propwords(username, lastWord, redisc)
if lastWord == "" || lastWord == "_END_" {
return outputMsg
}
if supressionUsernames[lastWord] {
if len(lastWord) < 2 {
lastWord = "vee"
}
lastWord = fmt.Sprintf("%s.%s", string(lastWord[0]), lastWord[1:])
}
outputMsg += lastWord + " "
if len(outputMsg) > 100 {
return outputMsg
}
}
}
func propwords(username string, start string, redisc *redis.Client) string {
userHash := redisc.HGetAll(fmt.Sprintf("%s-%s", username, start))
userHashMap, err := userHash.Result()
if err != nil {
genericHash := redisc.HGetAll(fmt.Sprintf("generic-%s", start))
userHashMap, err = genericHash.Result()
}
userIntHashMap, totalVectors := stringMaptoIntMap(userHashMap)
if totalVectors == 0 {
return ""
}
targetRand := rand.Intn(totalVectors)
progresRand := 0
for k, v := range userIntHashMap {
progresRand += v
if targetRand > progresRand {
return k
}
}
for k, _ := range userIntHashMap {
return k
}
return ""
}
func stringMaptoIntMap(in map[string]string) (outMap map[string]int, total int) {
outMap = make(map[string]int)
for k, v := range in {
i, err := strconv.ParseInt(v, 10, 64)
if err != nil {
continue
}
total += int(i)
outMap[k] = int(i)
}
return outMap, total
}
func learnFromMessage(msg incomingIRC, redisc *redis.Client) {
text := msg.Params[1]
text = strings.ToLower(text)
text = strings.Replace(text, ",", "", -1)
text = strings.Replace(text, ",", "", -1)
text = strings.Replace(text, ".", "", -1)
text = strings.Replace(text, "!", "", -1)
text = strings.Replace(text, "?", "", -1)
words := strings.Split(text, " ")
username := msg.Name
for k, word := range words {
// HINCRBY myhash field 1
nextWord := "_END_"
if len(words)-1 != k {
nextWord = words[k+1]
}
redisc.HIncrBy(fmt.Sprintf("%s-%s", username, word), nextWord, 1)
redisc.HIncrBy(fmt.Sprintf("generic-%s", word), nextWord, 1)
}
}

174
fun/tvl-ebooks/parse-logs/main.go Normal file
View file

@ -0,0 +1,174 @@
package main
import (
"bufio"
"fmt"
"log"
"math/rand"
"os"
"regexp"
"strconv"
"strings"
"github.com/go-redis/redis"
)
type incomingIRC struct {
Command string `json:"Command"`
Host string `json:"Host"`
Name string `json:"Name"`
Params []string `json:"Params"`
User string `json:"User"`
}
var quicklogMatch = regexp.MustCompile(`<(\w+)> (.+)`)
func main() {
redisc := redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("127.0.0.1:%d", 6379),
Password: "", // no password set
DB: 0, // use default DB
})
fireaway := make(chan incomingIRC, 10)
go func() {
f, err := os.Open("tvl.txt")
if err != nil {
log.Printf("aaa %v", err)
os.Exit(0)
}
bio := bufio.NewReader(f)
for {
line, _, err := bio.ReadLine()
if err != nil {
break
}
sline := string(line)
offset := strings.Index(sline, "]")
notime := sline[offset+1:]
if quicklogMatch.MatchString(notime) {
bits := quicklogMatch.FindAllStringSubmatch(notime, -1)
if len(bits) != 0 {
if len(bits[0]) != 0 {
a := make([]string, 2)
a[1] = bits[0][2]
ic := incomingIRC{
Name: bits[0][1],
Params: a,
}
log.Printf("aa %#v", ic)
fireaway <- ic
}
}
}
}
}()
for msg := range fireaway {
// Learn
learnFromMessage(msg, redisc)
// os.Exit(0)
}
}
func generateMesasge(msg incomingIRC, redisc *redis.Client) string {
text := msg.Params[1]
username := msg.Name
text = strings.ToLower(text)
text = strings.Replace(text, ",", "", -1)
text = strings.Replace(text, ",", "", -1)
text = strings.Replace(text, ".", "", -1)
text = strings.Replace(text, "!", "", -1)
text = strings.Replace(text, "?", "", -1)
words := strings.Split(text, " ")
lastWord := propwords(username, words[0], redisc)
outputMsg := words[0] + " " + lastWord + " "
for {
lastWord = propwords(username, words[0], redisc)
if lastWord == "" || lastWord == "_END_" {
return outputMsg
}
outputMsg += lastWord + " "
if len(outputMsg) > 100 {
return outputMsg
}
}
}
func propwords(username string, start string, redisc *redis.Client) string {
userHash := redisc.HGetAll(fmt.Sprintf("%s-%s", username, start))
userHashMap, err := userHash.Result()
if err != nil {
genericHash := redisc.HGetAll(fmt.Sprintf("generic-%s", start))
userHashMap, err = genericHash.Result()
}
userIntHashMap, totalVectors := stringMaptoIntMap(userHashMap)
targetRand := rand.Intn(totalVectors)
progresRand := 0
for k, v := range userIntHashMap {
progresRand += v
if targetRand > progresRand {
return k
}
}
for k, _ := range userIntHashMap {
return k
}
return ""
}
func stringMaptoIntMap(in map[string]string) (outMap map[string]int, total int) {
outMap = make(map[string]int)
for k, v := range in {
i, err := strconv.ParseInt(v, 10, 64)
if err != nil {
continue
}
total += int(i)
outMap[k] = int(i)
}
return outMap, total
}
func learnFromMessage(msg incomingIRC, redisc *redis.Client) {
text := msg.Params[1]
text = strings.ToLower(text)
text = strings.Replace(text, ",", "", -1)
text = strings.Replace(text, ",", "", -1)
text = strings.Replace(text, ".", "", -1)
text = strings.Replace(text, "!", "", -1)
text = strings.Replace(text, "?", "", -1)
words := strings.Split(text, " ")
username := msg.Name
for k, word := range words {
// HINCRBY myhash field 1
nextWord := "_END_"
if len(words)-1 != k {
nextWord = words[k+1]
}
redisc.HIncrBy(fmt.Sprintf("%s-%s", username, word), nextWord, 1)
redisc.HIncrBy(fmt.Sprintf("generic-%s", word), nextWord, 1)
}
}