type type type 📺

.github/workflows/build.yaml

@@ -0,0 +1,51 @@
+name: Building and Push to Docker Hub
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+env:
+  APP_IMAGE: lulis:${{ github.sha }}
+
+jobs:
+  build-and-push:
+    name: Building and Push Image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.21
+
+      - uses: actions/checkout@master
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Test
+        run: go test ./...
+
+      - name: Install doctl
+        if: github.ref == 'refs/heads/main'
+        uses: digitalocean/action-doctl@v2
+        with:
+          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+
+      - name: Log in to DO Container Registry
+        if: github.ref == 'refs/heads/main'
+        run: doctl registry login --expiry-seconds 600
+
+      - name: Tag image
+        if: github.ref == 'refs/heads/main'
+        run: |
+          docker build -f Dockerfile \
+            -t $APP_IMAGE \
+            -t registry.digitalocean.com/wallet-passes/$APP_IMAGE .
+
+      - name: Push image to DO Container Registry
+        if: github.ref == 'refs/heads/main'
+        run: |
+          docker push registry.digitalocean.com/wallet-passes/$APP_IMAGE

.gitignore

@@ -1,21 +1,11 @@
-# If you prefer the allow list template instead of the deny list, see community template:
-# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
-#
-# Binaries for programs and plugins
 *.exe
 *.exe~
 *.dll
 *.so
 *.dylib
-
-# Test binary, built with `go test -c`
 *.test
-
-# Output of the go coverage tool, specifically when used with LiteIDE
 *.out
-
-# Dependency directories (remove the comment below to include it)
-# vendor/
-
-# Go workspace file
 go.work
+.idea
+vendor
+tmp/*

Dockerfile

@@ -0,0 +1,29 @@
+FROM golang:1.21-bullseye
+
+# Set destination for COPY
+WORKDIR /app
+
+ENV CGO_ENABLED=0
+
+# Download Go modules
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY assets ./assets
+COPY cmd ./cmd
+COPY internal ./internal
+COPY assets ./assets
+
+RUN mkdir tmp && chmod 777 tmp
+
+RUN apt-get update
+RUN apt-get upgrade -y
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential libssl-dev ffmpeg
+
+# Build
+RUN GOOS=linux go build ./cmd/api/main.go
+
+EXPOSE 80
+
+# Run
+CMD ["/app/main"]

README.md

@@ -1,2 +1,40 @@
 # lulis
-Lula AI Stream
+
+Lulis is a simple PoC putting together FFMPEG, ChatGPT, Eleven Labs and Replicate.com to create a 24/7 AI Live Stream
+
+## Requirements
+
+```yaml
+- TWITCH_CHANNEL_NAME=${LULIS_TWITCH_CHANNEL_NAME}
+- TWITCH_STREAM_KEY=${LULIS_TWITCH_STREAM_KEY}
+- TWITCH_CLIENT_ID=${LULIS_TWITCH_CLIENT_ID}
+- OPEN_AI_KEY=${LULIS_OPEN_AI_KEY}
+- ELEVEN_LABS_KEY=${LULIS_ELEVEN_LABS_KEY}
+- ELEVEN_LABS_VOICE_ID=${LULIS_ELEVEN_LABS_VOICE_ID}
+- REPLICATE_KEY=${LULIS_REPLICATE_KEY}
+- AWS_BUCKET_NAME=${LULIS_AWS_BUCKET_NAME}
+- AWS_REGION=${LULIS_AWS_REGION}
+- AWS_ACCESS_KEY_ID=${LULIS_AWS_ACCESS_KEY_ID}
+- AWS_SECRET_ACCESS_KEY=${LULIS_AWS_SECRET_ACCESS_KEY}
+```
+
+Considering the following environment variables that we have to configure first, you can already see the list of services that we will have to prepare first:
+- Twitch Account (https://twitchapps.com/tmi/ use this for TWITCH_CLIENT_ID)
+- OpenAI Developer Account https://platform.openai.com/login?launch
+- Eleven Labs https://elevenlabs.io/ with a Cloned Voice for the ELEVEN_LABS_VOICE_ID
+- Replicate.com https://replicate.com/
+- AWS S3 https://aws.amazon.com/pm/serv-s3
+
+## Run
+
+```bash
+$ docker-compose up
+```
+
+## Test
+
+```bash
+$ go test ./...
+```
+
+No tests written yet 👹

assets/playlist.txt

@@ -0,0 +1,3 @@
+ffconcat version 1.0
+file 'loop.mp4'
+file 'temp_playlist.txt'

assets/temp_playlist.txt

@@ -0,0 +1,3 @@
+ffconcat version 1.0
+file 'loop.mp4'
+file 'playlist.txt'

cmd/api/main.go

@@ -0,0 +1,175 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gempir/go-twitch-irc/v4"
+	"github.com/llumus/lulis/internal/fs/s3"
+	"github.com/llumus/lulis/internal/gpt/openai"
+	"github.com/llumus/lulis/internal/mixer/replicate"
+	"github.com/llumus/lulis/internal/queue/memory"
+	"github.com/llumus/lulis/internal/stream/ffmpeg"
+	"github.com/llumus/lulis/internal/tts/elevenlabs"
+	"github.com/sirupsen/logrus"
+)
+
+var log = logrus.New()
+
+var bannedWords = []string{
+	"learning disability",
+	"learning disabilities",
+	"learning disorder",
+	"learning disorders",
+	"learning difficulties",
+	"learning difficulty",
+	"problemas de aprendizagem",
+	"problema de aprendizagem",
+	"transtorno de aprendizagem",
+	"replace",
+	"replaced",
+	"replacing",
+	"replaces",
+	"substitua",
+	"substituir",
+	"substituindo",
+	"\"",
+	"“",
+	"”",
+	"’",
+	"‘",
+	"''",
+	"``",
+	"\\",
+	"!*",
+}
+
+// healthCheckHandler is a simple HTTP handler function which writes a response used for cloud deploys
+func healthCheckHandler(w http.ResponseWriter, r *http.Request) {
+	// Send an 'OK' response with HTTP 200 status code
+	fmt.Fprintf(w, "OK")
+}
+
+func main() {
+	var port = os.Getenv("PORT")
+	var basePath = os.Getenv("BASE_PATH")
+	var twitchChannelName = os.Getenv("TWITCH_CHANNEL_NAME")
+	var twitchStreamKey = os.Getenv("TWITCH_STREAM_KEY")
+	var twitchClientId = os.Getenv("TWITCH_CLIENT_ID")
+	var openAiKey = os.Getenv("OPEN_AI_KEY")
+	var elevenLabsKey = os.Getenv("ELEVEN_LABS_KEY")
+	var elevenLabsVoiceId = os.Getenv("ELEVEN_LABS_VOICE_ID")
+	var replicateKey = os.Getenv("REPLICATE_KEY")
+	var awsBucket = os.Getenv("AWS_BUCKET_NAME")
+	var awsBaseUrl = os.Getenv("AWS_BUCKET_BASE_URL")
+	var faceVideoUrl = os.Getenv("FACE_VIDEO_URL")
+
+	gpt := openai.NewOpenAI(openAiKey)
+	fs := s3.NewFileSystem(awsBucket, basePath)
+	tts := elevenlabs.NewElevenLabs(elevenLabsKey, basePath, elevenLabsVoiceId, http.DefaultClient, fs)
+	stream := ffmpeg.NewStream(twitchStreamKey, filepath.Join(basePath, "tmp", "playlist.txt"))
+	mixer := replicate.NewMixer(replicateKey, awsBaseUrl, faceVideoUrl, http.DefaultClient, fs)
+
+	go func() {
+		http.HandleFunc("/", healthCheckHandler)
+		fmt.Println("Server is running on port " + port)
+		http.ListenAndServe(":"+port, nil)
+	}()
+
+	go func() {
+		log.Println("Starting stream...")
+		err := stream.StartStream()
+		if err != nil {
+			log.Println("Error starting stream:", err)
+			panic(err)
+		}
+	}()
+
+	videoQueue := memory.NewQueue()
+	go func() {
+		for {
+			video, ok := videoQueue.Dequeue()
+			if ok {
+				log.Infof("Video from queue: %s", video)
+
+				err := stream.PlayLatest(video)
+				if err != nil {
+					log.Errorf("Error switching video: %v", err)
+					continue
+				}
+			}
+		}
+	}()
+
+	msgQueue := memory.NewQueue()
+	go func() {
+		ctx := context.Background()
+		for {
+			message, ok := msgQueue.Dequeue()
+			if ok {
+				log.Debugf("Message from queue: %s", message)
+
+				var bannedWordDetected string
+				for _, word := range bannedWords {
+					if strings.Contains(message, word) {
+						bannedWordDetected = word
+						break
+					}
+				}
+
+				if bannedWordDetected != "" {
+					log.Warnf("Banned word detected: %s", bannedWordDetected)
+					continue
+				}
+
+				answer, err := gpt.GenerateResponse(ctx, message)
+				if err != nil {
+					log.Println("Error generating response:", err)
+					continue
+				}
+
+				log.Infof("Generated response for: %s", answer)
+				log.Infof("Generating audio for: %s", answer)
+
+				fsKey, err := tts.GenerateAudio(ctx, answer)
+				if err != nil {
+					log.Println("Error generating audio:", err)
+					continue
+				}
+
+				log.Infof("Generated audio: %s", fsKey)
+				log.Infof("Generating lip sync for: %s", answer)
+
+				videoLocalPath, err := mixer.GenerateLipSyncVideo(ctx, fsKey)
+				if err != nil {
+					log.Println("Error generating video:", err)
+					continue
+				}
+
+				log.Infof("Generated video: %s", videoLocalPath)
+				log.Infof("Sending video to queue: %s", videoLocalPath)
+
+				videoQueue.Enqueue(videoLocalPath)
+			}
+		}
+	}()
+
+	client := twitch.NewClient(twitchChannelName, twitchClientId)
+	client.Join(twitchChannelName)
+	client.OnPrivateMessage(func(message twitch.PrivateMessage) {
+		log.Infof("Message received: %s", message.Message)
+		if strings.HasPrefix(message.Message, "Lula, ") {
+			log.Infof("Message to the queue: %s", message.Message)
+			msgQueue.Enqueue(message.Message + " - " + message.User.Name)
+		}
+	})
+
+	err := client.Connect()
+	if err != nil {
+		panic(err)
+	}
+}

docker-compose.yaml

@@ -0,0 +1,29 @@
+version: '2.1'
+
+services:
+  lulis:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - "80:80"
+    container_name: lulis
+    volumes:
+      - ./tmp:/app/tmp
+    environment:
+      - LOG_LEVEL=debug
+      - PORT=80
+      - BASE_PATH=/app
+      - FACE_VIDEO_URL="https://lulis.s3.amazonaws.com/33.mp4"
+      - TWITCH_CHANNEL_NAME=${LULIS_TWITCH_CHANNEL_NAME}
+      - TWITCH_STREAM_KEY=${LULIS_TWITCH_STREAM_KEY}
+      - TWITCH_CLIENT_ID=${LULIS_TWITCH_CLIENT_ID}
+      - OPEN_AI_KEY=${LULIS_OPEN_AI_KEY}
+      - ELEVEN_LABS_KEY=${LULIS_ELEVEN_LABS_KEY}
+      - ELEVEN_LABS_VOICE_ID=${LULIS_ELEVEN_LABS_VOICE_ID}
+      - REPLICATE_KEY=${LULIS_REPLICATE_KEY}
+      - AWS_BUCKET_NAME=${LULIS_AWS_BUCKET_NAME}
+      - AWS_REGION=${LULIS_AWS_REGION}
+      - AWS_ACCESS_KEY_ID=${LULIS_AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${LULIS_AWS_SECRET_ACCESS_KEY}
+      - AWS_BUCKET_BASE_URL="https://lulis.s3.amazonaws.com/"

go.mod

@@ -0,0 +1,16 @@
+module github.com/llumus/lulis
+
+go 1.21.0
+
+require (
+	github.com/aws/aws-sdk-go v1.45.19
+	github.com/ayush6624/go-chatgpt v0.3.0
+	github.com/gempir/go-twitch-irc/v4 v4.0.0
+	github.com/google/uuid v1.3.1
+	github.com/sirupsen/logrus v1.9.3
+)
+
+require (
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	golang.org/x/sys v0.1.0 // indirect
+)