Skip to content

Commit

Permalink
cron: batch deletions of activity logs to workaround sqlite limitation
Browse files Browse the repository at this point in the history
Signed-off-by: Matthew Penner <[email protected]>
  • Loading branch information
matthewpi committed Jan 12, 2025
1 parent a55277d commit c6c235d
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 7 deletions.
23 changes: 18 additions & 5 deletions internal/cron/activity_cron.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ func (ac *activityCron) Run(ctx context.Context) error {
activities = append(activities, v)
}

// Delete any invalid activies
if len(ids) > 0 {
tx = database.Instance().WithContext(ctx).Where("id IN ?", ids).Delete(&models.Activity{})
if tx.Error != nil {
Expand All @@ -71,16 +72,28 @@ func (ac *activityCron) Run(ctx context.Context) error {
return errors.WrapIf(err, "cron: failed to send activity events to Panel")
}

// Add all the successful activities to the list of IDs to delete.
ids = make([]int, len(activities))
for i, v := range activities {
ids[i] = v.ID
}

// Delete all the activities that were sent to the Panel (or that were invalid).
tx = database.Instance().WithContext(ctx).Where("id IN ?", ids).Delete(&models.Activity{})
if tx.Error != nil {
return errors.WithStack(tx.Error)
// SQLite has a limitation of how many parameters we can specify in a single
// query, so we need to delete the activies in chunks of 32,000 instead of
// all at once.
i := 0
idsLen := len(ids)
for i < idsLen {
start := i
end := min(i+32000, idsLen)
batchSize := end - start

tx = database.Instance().WithContext(ctx).Where("id IN ?", ids[start:end]).Delete(&models.Activity{})
if tx.Error != nil {
return errors.WithStack(tx.Error)
}

i += batchSize
}

return nil
}
22 changes: 20 additions & 2 deletions internal/cron/sftp_cron.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"reflect"

"emperror.dev/errors"
"gorm.io/gorm"

"github.com/pterodactyl/wings/internal/database"
"github.com/pterodactyl/wings/internal/models"
Expand Down Expand Up @@ -83,9 +84,26 @@ func (sc *sftpCron) Run(ctx context.Context) error {
if err := sc.manager.Client().SendActivityLogs(ctx, events.Elements()); err != nil {
return errors.Wrap(err, "failed to send sftp activity logs to Panel")
}
if tx := database.Instance().Where("id IN ?", events.ids).Delete(&models.Activity{}); tx.Error != nil {
return errors.WithStack(tx.Error)

// SQLite has a limitation of how many parameters we can specify in a single
// query, so we need to delete the activies in chunks of 32,000 instead of
// all at once.
i := 0
idsLen := len(events.ids)
var tx *gorm.DB
for i < idsLen {
start := i
end := min(i+32000, idsLen)
batchSize := end - start

tx = database.Instance().WithContext(ctx).Where("id IN ?", events.ids[start:end]).Delete(&models.Activity{})
if tx.Error != nil {
return errors.WithStack(tx.Error)
}

i += batchSize
}

return nil
}

Expand Down

0 comments on commit c6c235d

Please sign in to comment.