From a9aa2177936018c4843dea60d0d43a0c213e694c Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Thu, 3 Oct 2024 20:48:51 +0200
Subject: [PATCH 01/16] events: do not derive Default on event sections

It is not required by the core or by any other piece of infrastructure.
Only some events need to derive from Default because of the way they are
constructed.

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 retis-derive/src/lib.rs          | 1 -
 retis-events/src/common.rs       | 1 +
 retis-events/src/kernel.rs       | 1 +
 retis-events/src/nft.rs          | 1 +
 retis-events/src/ovs.rs          | 2 +-
 retis-events/src/skb.rs          | 1 +
 retis-events/src/skb_tracking.rs | 2 +-
 retis/src/collect/collector.rs   | 1 +
 retis/src/core/events/bpf.rs     | 1 +
 9 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/retis-derive/src/lib.rs b/retis-derive/src/lib.rs
index 78186b41c..2dee16155 100644
--- a/retis-derive/src/lib.rs
+++ b/retis-derive/src/lib.rs
@@ -26,7 +26,6 @@ pub fn event_section(
     let id: syn::Expr = syn::parse(args).expect("Invalid event id");
 
     let output = quote! {
-        #[derive(Default)]
         #[crate::event_type]
         #input
 
diff --git a/retis-events/src/common.rs b/retis-events/src/common.rs
index 1da7bf5e8..d4a06f14e 100644
--- a/retis-events/src/common.rs
+++ b/retis-events/src/common.rs
@@ -34,6 +34,7 @@ pub struct TaskEvent {
 
 /// Common event section.
 #[event_section(SectionId::Common)]
+#[derive(Default)]
 pub struct CommonEvent {
     /// Timestamp of when the event was generated.
     pub timestamp: u64,
diff --git a/retis-events/src/kernel.rs b/retis-events/src/kernel.rs
index b9f280f97..965621f34 100644
--- a/retis-events/src/kernel.rs
+++ b/retis-events/src/kernel.rs
@@ -4,6 +4,7 @@ use super::*;
 use crate::{event_section, event_type, Formatter};
 
 #[event_section(SectionId::Kernel)]
+#[derive(Default)]
 pub struct KernelEvent {
     /// Kernel symbol name associated with the event (i.e. which probe generated
     /// the event).
diff --git a/retis-events/src/nft.rs b/retis-events/src/nft.rs
index c6c1e3ebd..2ee707615 100644
--- a/retis-events/src/nft.rs
+++ b/retis-events/src/nft.rs
@@ -5,6 +5,7 @@ use crate::{event_section, Formatter};
 
 /// Nft event section
 #[event_section(SectionId::Nft)]
+#[derive(Default)]
 pub struct NftEvent {
     pub table_name: String,
     pub chain_name: String,
diff --git a/retis-events/src/ovs.rs b/retis-events/src/ovs.rs
index a5b153669..d5637b3ae 100644
--- a/retis-events/src/ovs.rs
+++ b/retis-events/src/ovs.rs
@@ -7,8 +7,8 @@ use super::*;
 use crate::{event_section, event_type, Formatter};
 
 ///The OVS Event
-#[derive(PartialEq)]
 #[event_section(SectionId::Ovs)]
+#[derive(Default, PartialEq)]
 pub struct OvsEvent {
     /// Event data
     #[serde(flatten)]
diff --git a/retis-events/src/skb.rs b/retis-events/src/skb.rs
index b0ef7204f..ce0aa9f42 100644
--- a/retis-events/src/skb.rs
+++ b/retis-events/src/skb.rs
@@ -8,6 +8,7 @@ use crate::{event_section, event_type, Formatter};
 
 /// Skb event section.
 #[event_section(SectionId::Skb)]
+#[derive(Default)]
 pub struct SkbEvent {
     /// Ethernet fields, if any.
     pub eth: Option<SkbEthEvent>,
diff --git a/retis-events/src/skb_tracking.rs b/retis-events/src/skb_tracking.rs
index 620cc6ebd..7937f5cdc 100644
--- a/retis-events/src/skb_tracking.rs
+++ b/retis-events/src/skb_tracking.rs
@@ -14,8 +14,8 @@ use crate::{event_section, Formatter};
 ///
 /// Tl;dr; the tracking unique id is `(timestamp, orig_head)` and `skb` can be
 /// used to distinguished between clones.
-#[derive(Copy, PartialEq)]
 #[event_section(SectionId::SkbTracking)]
+#[derive(Default, Copy, PartialEq)]
 #[repr(C)]
 pub struct SkbTrackingEvent {
     /// Head of buffer (`skb->head`) when the packet was first seen by the
diff --git a/retis/src/collect/collector.rs b/retis/src/collect/collector.rs
index 69dff83e9..7aaad6b8a 100644
--- a/retis/src/collect/collector.rs
+++ b/retis/src/collect/collector.rs
@@ -612,6 +612,7 @@ mod tests {
     }
 
     #[event_section(SectionId::Common)]
+    #[derive(Default)]
     struct TestEvent {}
 
     impl EventFmt for TestEvent {
diff --git a/retis/src/core/events/bpf.rs b/retis/src/core/events/bpf.rs
index bff985d9a..268501d3d 100644
--- a/retis/src/core/events/bpf.rs
+++ b/retis/src/core/events/bpf.rs
@@ -644,6 +644,7 @@ mod tests {
     const DATA_TYPE_U128: u8 = 2;
 
     #[event_section(SectionId::Common)]
+    #[derive(Default)]
     struct TestEvent {
         field0: Option<u64>,
         field1: Option<u64>,

From ba010222717b216f84048e2861b1dc7454268dae Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Mon, 19 Aug 2024 11:23:16 +0200
Subject: [PATCH 02/16] events: ovs: Use Option instead of unit enum value.

OVS module currently makes no assumptions on the order of the event
chunks and tries to build the event even if out-of-order pieces are
received. This makes little sense in practice as these chuncks are sent
in the same hook.

Removing that unneeded requirement and assuming the base action event
(OVS_DP_ACTION) will be received before specific action argument events
(e.g: OVS_DP_ACTION_OUTPUT) makes decoding simpler.

This also it avoids requiring a Default version of the event which is
currently implemented using an "undefined" value for enums. This
mechanism is not supported by pyo3.

Also, create a dummy object to avoid having mixed complex unit variants
in enums.

[1] Upstream discussions:
https://github.com/PyO3/pyo3/pull/3582#discussion_r1443166018
https://github.com/PyO3/pyo3/issues/3749

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 retis-events/src/ovs.rs       | 116 ++++++++--------
 retis/src/module/ovs/bpf.rs   | 253 +++++++++++++++-------------------
 retis/src/process/tracking.rs |   1 -
 3 files changed, 168 insertions(+), 202 deletions(-)

diff --git a/retis-events/src/ovs.rs b/retis-events/src/ovs.rs
index d5637b3ae..1ea3d5a1a 100644
--- a/retis-events/src/ovs.rs
+++ b/retis-events/src/ovs.rs
@@ -8,7 +8,7 @@ use crate::{event_section, event_type, Formatter};
 
 ///The OVS Event
 #[event_section(SectionId::Ovs)]
-#[derive(Default, PartialEq)]
+#[derive(PartialEq)]
 pub struct OvsEvent {
     /// Event data
     #[serde(flatten)]
@@ -23,7 +23,7 @@ impl EventFmt for OvsEvent {
 
 #[event_type]
 #[serde(tag = "event_type")]
-#[derive(Default, PartialEq)]
+#[derive(PartialEq)]
 pub enum OvsEventType {
     /// Upcall event. It indicates the begining of an upcall. An upcall can have multiple enqueue
     /// events.
@@ -51,10 +51,6 @@ pub enum OvsEventType {
     /// Action execution event. It indicates the datapath has executed an action on a packet.
     #[serde(rename = "action_execute")]
     Action(ActionEvent),
-
-    #[serde(rename = "undefined")]
-    #[default]
-    Undefined,
 }
 
 impl EventFmt for OvsEventType {
@@ -67,7 +63,6 @@ impl EventFmt for OvsEventType {
             RecvUpcall(e) => e,
             Operation(e) => e,
             Action(e) => e,
-            Undefined => return write!(f, "?"),
         };
 
         disp.event_fmt(f, format)
@@ -266,7 +261,7 @@ impl EventFmt for RecvUpcallEvent {
 pub struct ActionEvent {
     /// Action to be executed.
     #[serde(flatten)]
-    pub action: OvsAction,
+    pub action: Option<OvsAction>,
     /// Recirculation id.
     pub recirc_id: u32,
     /// Queue id used for tracking. None if not tracking or if the output event did not come from
@@ -284,19 +279,18 @@ impl EventFmt for ActionEvent {
         write!(f, "exec")?;
 
         match &self.action {
-            OvsAction::Unspecified => write!(f, " (unspecified)")?,
-            OvsAction::Output(a) => write!(f, " oport {}", a.port)?,
-            OvsAction::Userspace => write!(f, " userspace")?,
-            OvsAction::Set => write!(f, " tunnel_set")?,
-            OvsAction::PushVlan => write!(f, " push_vlan")?,
-            OvsAction::PopVlan => write!(f, " pop_vlan")?,
-            OvsAction::Sample => write!(f, " sample")?,
-            OvsAction::Recirc(a) => write!(f, " recirc {:#x}", a.id)?,
-            OvsAction::Hash => write!(f, " hash")?,
-            OvsAction::PushMpls => write!(f, " push_mpls")?,
-            OvsAction::PopMpls => write!(f, " pop_mpls")?,
-            OvsAction::SetMasked => write!(f, " set_masked")?,
-            OvsAction::Ct(ct) => {
+            Some(OvsAction::Output(a)) => write!(f, " oport {}", a.port)?,
+            Some(OvsAction::Userspace(_)) => write!(f, " userspace")?,
+            Some(OvsAction::Set(_)) => write!(f, " tunnel_set")?,
+            Some(OvsAction::PushVlan(_)) => write!(f, " push_vlan")?,
+            Some(OvsAction::PopVlan(_)) => write!(f, " pop_vlan")?,
+            Some(OvsAction::Sample(_)) => write!(f, " sample")?,
+            Some(OvsAction::Recirc(a)) => write!(f, " recirc {:#x}", a.id)?,
+            Some(OvsAction::Hash(_)) => write!(f, " hash")?,
+            Some(OvsAction::PushMpls(_)) => write!(f, " push_mpls")?,
+            Some(OvsAction::PopMpls(_)) => write!(f, " pop_mpls")?,
+            Some(OvsAction::SetMasked(_)) => write!(f, " set_masked")?,
+            Some(OvsAction::Ct(ct)) => {
                 write!(f, " ct zone {}", ct.zone_id)?;
 
                 if let Some(nat) = &ct.nat {
@@ -358,17 +352,18 @@ impl EventFmt for ActionEvent {
                     write!(f, " {}", flags.join(","))?;
                 }
             }
-            OvsAction::Trunc => write!(f, " trunc")?,
-            OvsAction::PushEth => write!(f, " push_eth")?,
-            OvsAction::PopEth => write!(f, " pop_eth")?,
-            OvsAction::CtClear => write!(f, " ct_clear")?,
-            OvsAction::PushNsh => write!(f, " push_nsh")?,
-            OvsAction::PopNsh => write!(f, " pop_nsh")?,
-            OvsAction::Meter => write!(f, " meter")?,
-            OvsAction::Clone => write!(f, " clone")?,
-            OvsAction::CheckPktLen => write!(f, " check_pkt_len")?,
-            OvsAction::AddMpls => write!(f, " add_mpls")?,
-            OvsAction::DecTtl => write!(f, " dec_ttl")?,
+            Some(OvsAction::Trunc(_)) => write!(f, " trunc")?,
+            Some(OvsAction::PushEth(_)) => write!(f, " push_eth")?,
+            Some(OvsAction::PopEth(_)) => write!(f, " pop_eth")?,
+            Some(OvsAction::CtClear(_)) => write!(f, " ct_clear")?,
+            Some(OvsAction::PushNsh(_)) => write!(f, " push_nsh")?,
+            Some(OvsAction::PopNsh(_)) => write!(f, " pop_nsh")?,
+            Some(OvsAction::Meter(_)) => write!(f, " meter")?,
+            Some(OvsAction::Clone(_)) => write!(f, " clone")?,
+            Some(OvsAction::CheckPktLen(_)) => write!(f, " check_pkt_len")?,
+            Some(OvsAction::AddMpls(_)) => write!(f, " add_mpls")?,
+            Some(OvsAction::DecTtl(_)) => write!(f, " dec_ttl")?,
+            None => write!(f, " unspec")?,
         }
 
         if let Some(p) = self.queue_id {
@@ -379,59 +374,62 @@ impl EventFmt for ActionEvent {
     }
 }
 
+// Adding unit values in an otherwise complex is not supported by pyo3.
+// FIXME: Remove when arguments from all actions are implemented.
+#[event_type]
+#[derive(PartialEq)]
+pub struct OvsDummyAction;
+
 #[event_type]
 #[serde(tag = "action")]
-#[derive(Default, PartialEq)]
+#[derive(PartialEq)]
 pub enum OvsAction {
-    #[serde(rename = "unspecified")]
-    #[default]
-    Unspecified,
     #[serde(rename = "output")]
     Output(OvsActionOutput),
     #[serde(rename = "userspace")]
-    Userspace,
+    Userspace(OvsDummyAction),
     #[serde(rename = "set")]
-    Set,
+    Set(OvsDummyAction),
     #[serde(rename = "push_vlan")]
-    PushVlan,
+    PushVlan(OvsDummyAction),
     #[serde(rename = "pop_vlan")]
-    PopVlan,
+    PopVlan(OvsDummyAction),
     #[serde(rename = "sample")]
-    Sample,
+    Sample(OvsDummyAction),
     #[serde(rename = "recirc")]
     Recirc(OvsActionRecirc),
     #[serde(rename = "hash")]
-    Hash,
+    Hash(OvsDummyAction),
     #[serde(rename = "push_mpls")]
-    PushMpls,
+    PushMpls(OvsDummyAction),
     #[serde(rename = "pop_mpls")]
-    PopMpls,
+    PopMpls(OvsDummyAction),
     #[serde(rename = "set_masked")]
-    SetMasked,
+    SetMasked(OvsDummyAction),
     #[serde(rename = "ct")]
     Ct(OvsActionCt),
     #[serde(rename = "trunc")]
-    Trunc,
+    Trunc(OvsDummyAction),
     #[serde(rename = "push_eth")]
-    PushEth,
+    PushEth(OvsDummyAction),
     #[serde(rename = "pop_eth")]
-    PopEth,
+    PopEth(OvsDummyAction),
     #[serde(rename = "ct_clear")]
-    CtClear,
+    CtClear(OvsDummyAction),
     #[serde(rename = "push_nsh")]
-    PushNsh,
+    PushNsh(OvsDummyAction),
     #[serde(rename = "pop_nsh")]
-    PopNsh,
+    PopNsh(OvsDummyAction),
     #[serde(rename = "meter")]
-    Meter,
+    Meter(OvsDummyAction),
     #[serde(rename = "clone")]
-    Clone,
+    Clone(OvsDummyAction),
     #[serde(rename = "check_pkt_len")]
-    CheckPktLen,
+    CheckPktLen(OvsDummyAction),
     #[serde(rename = "add_mpls")]
-    AddMpls,
+    AddMpls(OvsDummyAction),
     #[serde(rename = "dec_ttl")]
-    DecTtl,
+    DecTtl(OvsDummyAction),
 }
 
 /// OVS output action data.
@@ -555,7 +553,7 @@ mod tests {
                 r#"{"action":"output","event_type":"action_execute","port":2,"queue_id":1361394472,"recirc_id":0}"#,
                 OvsEvent {
                     event: OvsEventType::Action(ActionEvent {
-                        action: OvsAction::Output(OvsActionOutput { port: 2 }),
+                        action: Some(OvsAction::Output(OvsActionOutput { port: 2 })),
                         recirc_id: 0,
                         queue_id: Some(1361394472),
                     }),
@@ -615,7 +613,7 @@ mod tests {
                 r#"{"action":"ct","event_type":"action_execute","flags":485,"nat":{"dir":"dst","max_addr":"10.244.1.30","max_port":36900,"min_addr":"10.244.1.3","min_port":36895},"recirc_id":34,"zone_id":20}"#,
                 OvsEvent {
                     event: OvsEventType::Action(ActionEvent {
-                        action: OvsAction::Ct(OvsActionCt {
+                        action: Some(OvsAction::Ct(OvsActionCt {
                             zone_id: 20,
                             flags: 485,
                             nat: Some(OvsActionCtNat {
@@ -625,7 +623,7 @@ mod tests {
                                 min_port: Some(36895),
                                 max_port: Some(36900),
                             }),
-                        }),
+                        })),
                         recirc_id: 34,
                         queue_id: None,
                     }),
diff --git a/retis/src/module/ovs/bpf.rs b/retis/src/module/ovs/bpf.rs
index 9f67f4a74..5c4e8c34a 100644
--- a/retis/src/module/ovs/bpf.rs
+++ b/retis/src/module/ovs/bpf.rs
@@ -4,7 +4,7 @@
 
 use std::net::Ipv6Addr;
 
-use anyhow::{bail, Result};
+use anyhow::{anyhow, bail, Result};
 
 use crate::{
     core::events::{
@@ -59,26 +59,11 @@ impl OvsDataType {
     }
 }
 
-// OVS module supports several event data types but many of them end up setting the OvsEvent
-// to one of its variants which mean they are mutally exclusive.
-// This helper ensures the event has was not set before to any of its variants to help
-// report his error condition.
-pub(crate) fn ensure_undefined(event: &OvsEvent, received: OvsDataType) -> Result<()> {
-    match &event.event {
-        OvsEventType::Undefined => Ok(()),
-        other => bail!(
-            "Conflicting OVS event types. Received {:?} data type but event is already {:#?}",
-            received,
-            other
-        ),
-    }
-}
-
-pub(super) fn unmarshall_upcall(raw_section: &BpfRawSection, event: &mut OvsEvent) -> Result<()> {
-    ensure_undefined(event, OvsDataType::Upcall)?;
+pub(super) fn unmarshall_upcall(raw_section: &BpfRawSection) -> Result<OvsEvent> {
     let upcall = parse_raw_section::<UpcallEvent>(raw_section)?;
-    event.event = OvsEventType::Upcall(*upcall);
-    Ok(())
+    Ok(OvsEvent {
+        event: OvsEventType::Upcall(*upcall),
+    })
 }
 
 /// OVS action event data.
@@ -90,68 +75,50 @@ pub(crate) struct BpfActionEvent {
     recirc_id: u32,
 }
 
-pub(super) fn unmarshall_exec(raw_section: &BpfRawSection, event: &mut OvsEvent) -> Result<()> {
+pub(super) fn unmarshall_exec(raw_section: &BpfRawSection) -> Result<OvsEvent> {
     let raw = parse_raw_section::<BpfActionEvent>(raw_section)?;
 
-    // Any of the action-related bpf events (e.g BpfActionTrackEvent, BpfActionTrackEvent, etc)
-    // might have been received before. If so, event.event is already a valid
-    // OvsEventType::Action.
-    match &mut event.event {
-        OvsEventType::Action(action) => {
-            // One of the specific action events has already been received and it has initialized
-            // the action.data enum. Only the common data has to be set here.
-            action.recirc_id = raw.recirc_id;
-        }
-        OvsEventType::Undefined => {
-            // When we implement event data types for every action we will be able to create the
-            // specific action variant when unmarshaling its event data type. Until then, we need to
-            // initialize the Action here based on the action_id (which corresponds to ovs_action_attr
-            // defined in uapi/linux/openvswitch.h).
-            event.event = OvsEventType::Action(ActionEvent {
-                action: match raw.action {
-                    0 => OvsAction::Unspecified,
-                    1 => OvsAction::Output(OvsActionOutput::default()),
-                    2 => OvsAction::Userspace,
-                    3 => OvsAction::Set,
-                    4 => OvsAction::PushVlan,
-                    5 => OvsAction::PopVlan,
-                    6 => OvsAction::Sample,
-                    7 => OvsAction::Recirc(OvsActionRecirc::default()),
-                    8 => OvsAction::Hash,
-                    9 => OvsAction::PushMpls,
-                    10 => OvsAction::PopMpls,
-                    11 => OvsAction::SetMasked,
-                    12 => OvsAction::Ct(OvsActionCt::default()),
-                    13 => OvsAction::Trunc,
-                    14 => OvsAction::PushEth,
-                    15 => OvsAction::PopEth,
-                    16 => OvsAction::CtClear,
-                    17 => OvsAction::PushNsh,
-                    18 => OvsAction::PopNsh,
-                    19 => OvsAction::Meter,
-                    20 => OvsAction::Clone,
-                    21 => OvsAction::CheckPktLen,
-                    22 => OvsAction::AddMpls,
-                    23 => OvsAction::DecTtl,
-                    // The private OVS_ACTION_ATTR_SET_TO_MASKED action is used
-                    // in the same way as OVS_ACTION_ATTR_SET_MASKED. Use only
-                    // one action to avoid confusion
-                    25 => OvsAction::SetMasked,
-                    val => bail!("Unsupported action id {val}"),
-                },
-                recirc_id: raw.recirc_id,
-                ..ActionEvent::default()
-            });
-        }
-        other => {
-            bail!(
-                "Conflicting OVS event types. Received {:?} data type but event is already {:#?}",
-                OvsDataType::ActionExec,
-                other
-            );
-        }
-    }
-    Ok(())
+    // When we implement event data types for every action we will be able to create the
+    // specific action variant when unmarshaling its event data type. Until then, we need to
+    // initialize the Action here based on the action_id (which corresponds to ovs_action_attr
+    // defined in uapi/linux/openvswitch.h).
+    Ok(OvsEvent {
+        event: OvsEventType::Action(ActionEvent {
+            action: match raw.action {
+                0 => None,
+                1 => Some(OvsAction::Output(OvsActionOutput::default())),
+                2 => Some(OvsAction::Userspace(OvsDummyAction)),
+                3 => Some(OvsAction::Set(OvsDummyAction)),
+                4 => Some(OvsAction::PushVlan(OvsDummyAction)),
+                5 => Some(OvsAction::PopVlan(OvsDummyAction)),
+                6 => Some(OvsAction::Sample(OvsDummyAction)),
+                7 => Some(OvsAction::Recirc(OvsActionRecirc::default())),
+                8 => Some(OvsAction::Hash(OvsDummyAction)),
+                9 => Some(OvsAction::PushMpls(OvsDummyAction)),
+                10 => Some(OvsAction::PopMpls(OvsDummyAction)),
+                11 => Some(OvsAction::SetMasked(OvsDummyAction)),
+                12 => Some(OvsAction::Ct(OvsActionCt::default())),
+                13 => Some(OvsAction::Trunc(OvsDummyAction)),
+                14 => Some(OvsAction::PushEth(OvsDummyAction)),
+                15 => Some(OvsAction::PopEth(OvsDummyAction)),
+                16 => Some(OvsAction::CtClear(OvsDummyAction)),
+                17 => Some(OvsAction::PushNsh(OvsDummyAction)),
+                18 => Some(OvsAction::PopNsh(OvsDummyAction)),
+                19 => Some(OvsAction::Meter(OvsDummyAction)),
+                20 => Some(OvsAction::Clone(OvsDummyAction)),
+                21 => Some(OvsAction::CheckPktLen(OvsDummyAction)),
+                22 => Some(OvsAction::AddMpls(OvsDummyAction)),
+                23 => Some(OvsAction::DecTtl(OvsDummyAction)),
+                // The private OVS_ACTION_ATTR_SET_TO_MASKED action is used
+                // in the same way as OVS_ACTION_ATTR_SET_MASKED. Use only
+                // one action to avoid confusion
+                25 => Some(OvsAction::SetMasked(OvsDummyAction)),
+                val => bail!("Unsupported action id {val}"),
+            },
+            recirc_id: raw.recirc_id,
+            ..ActionEvent::default()
+        }),
+    })
 }
 
 /// OVS action tracking event data.
@@ -167,19 +134,8 @@ pub(super) fn unmarshall_exec_track(
 ) -> Result<()> {
     let raw = parse_raw_section::<BpfActionTrackEvent>(raw_section)?;
 
-    // Any of the action-related bpf events (e.g BpfActionEvent, BpfActionTrackEvent, etc)
-    // might have been received before. If so, event.event is already a valid
-    // OvsEventType::Action.
     match &mut event.event {
         OvsEventType::Action(ref mut action) => action.queue_id = Some(raw.queue_id),
-        OvsEventType::Undefined => {
-            // We received the tracking event before the generic one.
-            // Initialize the Action Event.
-            event.event = OvsEventType::Action(ActionEvent {
-                queue_id: Some(raw.queue_id),
-                ..ActionEvent::default()
-            });
-        }
         other => {
             bail!(
                 "Conflicting OVS event types. Received {:?} data type but event is already {:#?}",
@@ -192,19 +148,8 @@ pub(super) fn unmarshall_exec_track(
 }
 
 fn update_action_event(event: &mut OvsEvent, action: OvsAction) -> Result<()> {
-    // Any of the action-related bpf events (e.g BpfActionEvent, BpfActionTrackEvent, etc)
-    // might have been received before. If so, event.event is already a valid
-    // OvsEventType::Action.
     match &mut event.event {
-        OvsEventType::Action(ref mut event) => event.action = action,
-        OvsEventType::Undefined => {
-            // We received the concrete action data type before the generic one.
-            // Initialize the Action Event.
-            event.event = OvsEventType::Action(ActionEvent {
-                action,
-                ..ActionEvent::default()
-            });
-        }
+        OvsEventType::Action(ref mut event) => event.action = Some(action),
         other => {
             bail!(
                 "Conflicting OVS event types. Received {:?} data type but event is already {:#?}",
@@ -308,45 +253,35 @@ pub(super) fn unmarshall_ct(raw_section: &BpfRawSection, event: &mut OvsEvent) -
     update_action_event(event, OvsAction::Ct(ct))
 }
 
-pub(super) fn unmarshall_recv(raw_section: &BpfRawSection, event: &mut OvsEvent) -> Result<()> {
-    ensure_undefined(event, OvsDataType::RecvUpcall)?;
+pub(super) fn unmarshall_recv(raw_section: &BpfRawSection) -> Result<OvsEvent> {
     let recv = parse_raw_section::<RecvUpcallEvent>(raw_section)?;
-    event.event = OvsEventType::RecvUpcall(*recv);
-
-    Ok(())
+    Ok(OvsEvent {
+        event: OvsEventType::RecvUpcall(*recv),
+    })
 }
 
-pub(super) fn unmarshall_operation(
-    raw_section: &BpfRawSection,
-    event: &mut OvsEvent,
-) -> Result<()> {
-    ensure_undefined(event, OvsDataType::Operation)?;
+pub(super) fn unmarshall_operation(raw_section: &BpfRawSection) -> Result<OvsEvent> {
     let op = parse_raw_section::<OperationEvent>(raw_section)?;
 
-    event.event = OvsEventType::Operation(*op);
-    Ok(())
+    Ok(OvsEvent {
+        event: OvsEventType::Operation(*op),
+    })
 }
 
-pub(super) fn unmarshall_upcall_enqueue(
-    raw_section: &BpfRawSection,
-    event: &mut OvsEvent,
-) -> Result<()> {
-    ensure_undefined(event, OvsDataType::UpcallEnqueue)?;
+pub(super) fn unmarshall_upcall_enqueue(raw_section: &BpfRawSection) -> Result<OvsEvent> {
     let enqueue = parse_raw_section::<UpcallEnqueueEvent>(raw_section)?;
 
-    event.event = OvsEventType::UpcallEnqueue(*enqueue);
-    Ok(())
+    Ok(OvsEvent {
+        event: OvsEventType::UpcallEnqueue(*enqueue),
+    })
 }
 
-pub(super) fn unmarshall_upcall_return(
-    raw_section: &BpfRawSection,
-    event: &mut OvsEvent,
-) -> Result<()> {
-    ensure_undefined(event, OvsDataType::UpcallReturn)?;
+pub(super) fn unmarshall_upcall_return(raw_section: &BpfRawSection) -> Result<OvsEvent> {
     let uret = parse_raw_section::<UpcallReturnEvent>(raw_section)?;
 
-    event.event = OvsEventType::UpcallReturn(*uret);
-    Ok(())
+    Ok(OvsEvent {
+        event: OvsEventType::UpcallReturn(*uret),
+    })
 }
 
 #[event_section_factory(FactoryId::Ovs)]
@@ -355,24 +290,58 @@ pub(crate) struct OvsEventFactory {}
 
 impl RawEventSectionFactory for OvsEventFactory {
     fn create(&mut self, raw_sections: Vec<BpfRawSection>) -> Result<Box<dyn EventSection>> {
-        let mut event = OvsEvent::default();
+        let mut event = None; // = OvsEvent::default();
 
         for section in raw_sections.iter() {
             match OvsDataType::from_u8(section.header.data_type)? {
-                OvsDataType::Upcall => unmarshall_upcall(section, &mut event),
-                OvsDataType::UpcallEnqueue => unmarshall_upcall_enqueue(section, &mut event),
-                OvsDataType::UpcallReturn => unmarshall_upcall_return(section, &mut event),
-                OvsDataType::RecvUpcall => unmarshall_recv(section, &mut event),
-                OvsDataType::Operation => unmarshall_operation(section, &mut event),
-                OvsDataType::ActionExec => unmarshall_exec(section, &mut event),
-                OvsDataType::ActionExecTrack => unmarshall_exec_track(section, &mut event),
-                OvsDataType::OutputAction => unmarshall_output(section, &mut event),
-                OvsDataType::RecircAction => unmarshall_recirc(section, &mut event),
-                OvsDataType::ConntrackAction => unmarshall_ct(section, &mut event),
-            }?;
+                OvsDataType::Upcall => {
+                    event = Some(unmarshall_upcall(section)?);
+                }
+                OvsDataType::UpcallEnqueue => {
+                    event = Some(unmarshall_upcall_enqueue(section)?);
+                }
+                OvsDataType::UpcallReturn => {
+                    event = Some(unmarshall_upcall_return(section)?);
+                }
+                OvsDataType::RecvUpcall => {
+                    event = Some(unmarshall_recv(section)?);
+                }
+                OvsDataType::Operation => {
+                    event = Some(unmarshall_operation(section)?);
+                }
+                OvsDataType::ActionExec => {
+                    event = Some(unmarshall_exec(section)?);
+                }
+                OvsDataType::ActionExecTrack => unmarshall_exec_track(
+                    section,
+                    event
+                        .as_mut()
+                        .ok_or_else(|| anyhow!("received action track without action"))?,
+                )?,
+                OvsDataType::OutputAction => unmarshall_output(
+                    section,
+                    event
+                        .as_mut()
+                        .ok_or_else(|| anyhow!("received action data without action"))?,
+                )?,
+                OvsDataType::RecircAction => unmarshall_recirc(
+                    section,
+                    event
+                        .as_mut()
+                        .ok_or_else(|| anyhow!("received action data without action"))?,
+                )?,
+                OvsDataType::ConntrackAction => unmarshall_ct(
+                    section,
+                    event
+                        .as_mut()
+                        .ok_or_else(|| anyhow!("received action data without action"))?,
+                )?,
+            };
         }
 
-        Ok(Box::new(event))
+        Ok(Box::new(
+            event.ok_or_else(|| anyhow!("Incomplete OVS event"))?,
+        ))
     }
 }
 
diff --git a/retis/src/process/tracking.rs b/retis/src/process/tracking.rs
index 15e2d96d6..0d7dfbd2b 100644
--- a/retis/src/process/tracking.rs
+++ b/retis/src/process/tracking.rs
@@ -133,7 +133,6 @@ impl AddTracking {
                         self.process_skb(event)?;
                     }
                 },
-                Undefined => bail!("Cannot track undefined ovs event"),
             }
         } else {
             // It's not an OVS event, try skb-only tracking.

From ce8cef47edfb3cc21c86b5c62428bd2e4f4b8630 Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Wed, 18 Sep 2024 18:03:07 +0200
Subject: [PATCH 03/16] events: move TestEvent to event crate.

It is useful to be able to define a dummy event to test their creation
(factories) and manipulation.

However, in preparation to add python support, we must ensure pyo3
dependencies only come from retis-events. Therefore, all events must be
defined there, including TestEvent.

Create a feature to enable this dummy event that is enabled only when
"cargo test" is run on the main crate.

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 retis-events/Cargo.toml        |  3 +++
 retis-events/src/events.rs     | 27 +++++++++++++++++++++++++++
 retis/Cargo.toml               |  1 +
 retis/src/collect/collector.rs | 17 ++++-------------
 retis/src/core/events/bpf.rs   | 21 ++-------------------
 5 files changed, 37 insertions(+), 32 deletions(-)

diff --git a/retis-events/Cargo.toml b/retis-events/Cargo.toml
index 42f1768ee..c9505e83a 100644
--- a/retis-events/Cargo.toml
+++ b/retis-events/Cargo.toml
@@ -3,6 +3,9 @@ name = "retis-events"
 version = "1.4.0"
 edition = "2021"
 
+[features]
+test-events = []
+
 [lib]
 crate-type = ["lib"]
 
diff --git a/retis-events/src/events.rs b/retis-events/src/events.rs
index 353c94204..dce4d89dc 100644
--- a/retis-events/src/events.rs
+++ b/retis-events/src/events.rs
@@ -328,3 +328,30 @@ impl EventSectionInternal for () {
         serde_json::Value::Null
     }
 }
+
+#[cfg(feature = "test-events")]
+pub mod test {
+    use super::*;
+    use crate::event_section;
+
+    #[event_section(SectionId::Common)]
+    #[derive(Default)]
+    pub struct TestEvent {
+        pub field0: Option<u64>,
+        pub field1: Option<u64>,
+        pub field2: Option<u64>,
+    }
+
+    impl EventFmt for TestEvent {
+        fn event_fmt(&self, f: &mut Formatter, _: &DisplayFormat) -> std::fmt::Result {
+            write!(
+                f,
+                "field0: {:?} field1: {:?} field2: {:?}",
+                self.field0, self.field1, self.field2
+            )
+        }
+    }
+}
+
+#[cfg(feature = "test-events")]
+pub use test::*;
diff --git a/retis/Cargo.toml b/retis/Cargo.toml
index c9fe2e806..76ef2fbd9 100644
--- a/retis/Cargo.toml
+++ b/retis/Cargo.toml
@@ -69,3 +69,4 @@ memmap2 = "0.9"
 probe = "0.5"
 serial_test = "3.1"
 test-case = "3.2"
+events = {version = "1.4", path = "../retis-events", package="retis-events", features = ["test-events"]}
diff --git a/retis/src/collect/collector.rs b/retis/src/collect/collector.rs
index 7aaad6b8a..3cf36a8b5 100644
--- a/retis/src/collect/collector.rs
+++ b/retis/src/collect/collector.rs
@@ -539,7 +539,7 @@ mod tests {
     use super::*;
     use crate::{
         core::{events::bpf::*, probe::ProbeBuilderManager},
-        event_section, event_section_factory,
+        event_section_factory,
         module::Module,
     };
 
@@ -575,7 +575,7 @@ mod tests {
             self
         }
         fn section_factory(&self) -> Result<Option<Box<dyn EventSectionFactory>>> {
-            Ok(Some(Box::new(TestEventFactory {})))
+            Ok(Some(Box::new(TestEventFactory::default())))
         }
     }
 
@@ -607,21 +607,12 @@ mod tests {
             self
         }
         fn section_factory(&self) -> Result<Option<Box<dyn EventSectionFactory>>> {
-            Ok(Some(Box::new(TestEventFactory {})))
-        }
-    }
-
-    #[event_section(SectionId::Common)]
-    #[derive(Default)]
-    struct TestEvent {}
-
-    impl EventFmt for TestEvent {
-        fn event_fmt(&self, f: &mut Formatter, _: &DisplayFormat) -> std::fmt::Result {
-            write!(f, "test event section")
+            Ok(Some(Box::new(TestEventFactory::default())))
         }
     }
 
     #[event_section_factory(FactoryId::Common)]
+    #[derive(Default)]
     struct TestEventFactory {}
 
     impl RawEventSectionFactory for TestEventFactory {
diff --git a/retis/src/core/events/bpf.rs b/retis/src/core/events/bpf.rs
index 268501d3d..8d7c15d69 100644
--- a/retis/src/core/events/bpf.rs
+++ b/retis/src/core/events/bpf.rs
@@ -638,29 +638,12 @@ mod tests {
     use serde::{Deserialize, Serialize};
 
     use super::*;
-    use crate::{event_section, event_section_factory};
+    use crate::event_section_factory;
+    use crate::events::TestEvent;
 
     const DATA_TYPE_U64: u8 = 1;
     const DATA_TYPE_U128: u8 = 2;
 
-    #[event_section(SectionId::Common)]
-    #[derive(Default)]
-    struct TestEvent {
-        field0: Option<u64>,
-        field1: Option<u64>,
-        field2: Option<u64>,
-    }
-
-    impl EventFmt for TestEvent {
-        fn event_fmt(&self, f: &mut Formatter, _: &DisplayFormat) -> std::fmt::Result {
-            write!(
-                f,
-                "field0: {:?} field1: {:?} field2: {:?}",
-                self.field0, self.field1, self.field2
-            )
-        }
-    }
-
     #[event_section_factory(FactoryId::Common)]
     #[derive(Default)]
     struct TestEventFactory {}

From cb81d644a797c8d9117c0fd82d65b66b1cfad6be Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Wed, 21 Aug 2024 10:20:06 +0200
Subject: [PATCH 04/16] events: move EventSeries to events crate.

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 retis-events/src/events.rs   | 13 +++++++++++++
 retis/src/process/display.rs |  1 -
 retis/src/process/series.rs  | 15 +--------------
 3 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/retis-events/src/events.rs b/retis-events/src/events.rs
index dce4d89dc..73daea6ab 100644
--- a/retis-events/src/events.rs
+++ b/retis-events/src/events.rs
@@ -329,6 +329,19 @@ impl EventSectionInternal for () {
     }
 }
 
+/// A set of sorted Events with the same tracking id.
+#[derive(Default)]
+pub struct EventSeries {
+    /// Events that comprise the Series.
+    pub events: Vec<Event>,
+}
+
+impl EventSeries {
+    pub fn to_json(&self) -> serde_json::Value {
+        serde_json::Value::Array(self.events.iter().map(|e| e.to_json()).collect())
+    }
+}
+
 #[cfg(feature = "test-events")]
 pub mod test {
     use super::*;
diff --git a/retis/src/process/display.rs b/retis/src/process/display.rs
index 25db81bac..c2b7cd908 100644
--- a/retis/src/process/display.rs
+++ b/retis/src/process/display.rs
@@ -2,7 +2,6 @@ use std::io::Write;
 
 use anyhow::Result;
 
-use super::series::EventSeries;
 use crate::events::*;
 
 /// Select the format to follow when printing events with `PrintEvent`.
diff --git a/retis/src/process/series.rs b/retis/src/process/series.rs
index 5c35871c4..4c3c44c13 100644
--- a/retis/src/process/series.rs
+++ b/retis/src/process/series.rs
@@ -9,20 +9,7 @@ use std::collections::{BTreeMap, VecDeque};
 
 use anyhow::{anyhow, Result};
 
-use crate::events::{CommonEvent, Event, SectionId, TrackingInfo};
-
-/// A set of sorted Events with the same tracking id.
-#[derive(Default)]
-pub(crate) struct EventSeries {
-    /// Events that comprise the Series.
-    pub(crate) events: Vec<Event>,
-}
-
-impl EventSeries {
-    pub(crate) fn to_json(&self) -> serde_json::Value {
-        serde_json::Value::Array(self.events.iter().map(|e| e.to_json()).collect())
-    }
-}
+use crate::events::{CommonEvent, Event, EventSeries, SectionId, TrackingInfo};
 
 #[derive(Default)]
 pub(crate) struct EventSorter {

From 42086d2df67fd55365feef8fb53799236604c44d Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Thu, 29 Aug 2024 21:19:50 +0200
Subject: [PATCH 05/16] events: make events implement Send

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 retis-events/src/events.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/retis-events/src/events.rs b/retis-events/src/events.rs
index 73daea6ab..e3fc9e8b0 100644
--- a/retis-events/src/events.rs
+++ b/retis-events/src/events.rs
@@ -295,8 +295,8 @@ pub enum EventResult {
 /// having a proper structure is encouraged as it allows easier consumption at
 /// post-processing. Those objects can also define their own specialized
 /// helpers.
-pub trait EventSection: EventSectionInternal + for<'a> EventDisplay<'a> {}
-impl<T> EventSection for T where T: EventSectionInternal + for<'a> EventDisplay<'a> {}
+pub trait EventSection: EventSectionInternal + for<'a> EventDisplay<'a> + Send {}
+impl<T> EventSection for T where T: EventSectionInternal + for<'a> EventDisplay<'a> + Send {}
 
 /// EventSection helpers defined in the core for all events. Common definition
 /// needs Sized but that is a requirement for all EventSection.

From 417373cad6f2a38ace496d9151b0321c1466d269 Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Tue, 1 Oct 2024 23:04:08 +0200
Subject: [PATCH 06/16] helpers: bimap: explicitly name the lifetime

It would anyway be resolved to 'a and nightly toolchain prefers it to be
explicit.

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 retis/src/helpers/bimap.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/retis/src/helpers/bimap.rs b/retis/src/helpers/bimap.rs
index 15a52999b..0ed65f512 100644
--- a/retis/src/helpers/bimap.rs
+++ b/retis/src/helpers/bimap.rs
@@ -77,7 +77,7 @@ where
 
     /// Creates an iterator over the key->value pairs within a range of keys in the bimap in
     /// ascending order.
-    pub(crate) fn range_by_left<'a, R>(&'a self, target: &'a R) -> KeyRange<'_, K, V>
+    pub(crate) fn range_by_left<'a, R>(&'a self, target: &'a R) -> KeyRange<'a, K, V>
     where
         Arc<K>: Borrow<K>,
         R: RangeBounds<K>,

From 0174e46eb7ce306f2c50981667b7750adda2d745 Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Wed, 16 Oct 2024 17:15:11 +0200
Subject: [PATCH 07/16] events: simplify EventFactory API.

It currently uses a struct EventResult that was originally thought to be
used when creating events from ebpf as it has a timeout. The file
factory is not using this so we can express the result as
Result<Option<Event>>.

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 retis-events/src/events.rs           | 10 ----------
 retis-events/src/file.rs             | 13 ++++++-------
 retis/src/benchmark/events_output.rs |  6 +++---
 retis/src/collect/collector.rs       |  3 +--
 retis/src/core/events/bpf.rs         |  8 ++++++++
 retis/src/process/cli/pcap.rs        |  9 +++------
 retis/src/process/cli/print.rs       | 10 ++++------
 retis/src/process/cli/sort.rs        |  7 +++----
 8 files changed, 28 insertions(+), 38 deletions(-)

diff --git a/retis-events/src/events.rs b/retis-events/src/events.rs
index e3fc9e8b0..e197876c5 100644
--- a/retis-events/src/events.rs
+++ b/retis-events/src/events.rs
@@ -273,16 +273,6 @@ fn event_sections() -> Result<&'static EventSectionMap> {
     })
 }
 
-/// The return value of EventFactory::next_event()
-pub enum EventResult {
-    /// The Factory was able to create a new event.
-    Event(Event),
-    /// The source has been consumed.
-    Eof,
-    /// The timeout went off but a new attempt to retrieve an event might succeed.
-    Timeout,
-}
-
 /// Per-module event section, should map 1:1 with a SectionId. Requiring specific
 /// traits to be implemented helps handling those sections in the core directly
 /// without requiring all modules to serialize and deserialize their events by
diff --git a/retis-events/src/file.rs b/retis-events/src/file.rs
index adfbc68ee..387cf3db5 100644
--- a/retis-events/src/file.rs
+++ b/retis-events/src/file.rs
@@ -4,12 +4,11 @@ use std::{
     fs::File,
     io::{BufRead, BufReader},
     path::Path,
-    time::Duration,
 };
 
 use anyhow::{anyhow, Result};
 
-use super::{Event, EventResult};
+use super::Event;
 
 /// File events factory retrieving and unmarshaling events
 /// parts.
@@ -32,17 +31,17 @@ impl FileEventsFactory {
 }
 
 impl FileEventsFactory {
-    /// Retrieve the next event. This is a blocking call.
-    pub fn next_event(&mut self, _timeout: Option<Duration>) -> Result<EventResult> {
+    /// Retrieve the next event or None if we've reached the end of the file.
+    pub fn next_event(&mut self) -> Result<Option<Event>> {
         let mut line = String::new();
 
         match self.reader.read_line(&mut line) {
             Err(e) => return Err(e.into()),
-            Ok(0) => return Ok(EventResult::Eof),
+            Ok(0) => return Ok(None),
             Ok(_) => (),
         }
 
-        Ok(EventResult::Event(Event::from_json(line)?))
+        Ok(Some(Event::from_json(line)?))
     }
 }
 
@@ -54,7 +53,7 @@ mod tests {
         let mut fact = FileEventsFactory::new("test_data/test_events.json").unwrap();
 
         let mut events = Vec::new();
-        while let EventResult::Event(event) = fact.next_event(None).unwrap() {
+        while let Some(event) = fact.next_event().unwrap() {
             println!("event: {:#?}", event.to_json());
             events.push(event)
         }
diff --git a/retis/src/benchmark/events_output.rs b/retis/src/benchmark/events_output.rs
index f267f3480..3ef8b2da1 100644
--- a/retis/src/benchmark/events_output.rs
+++ b/retis/src/benchmark/events_output.rs
@@ -17,8 +17,8 @@ pub(super) fn bench(ci: bool) -> Result<()> {
     // PrintEvent benchmark
 
     let mut factory = FileEventsFactory::new("retis/test_data/test_events_bench.json")?;
-    let event = match factory.next_event(None)? {
-        EventResult::Event(event) => event,
+    let event = match factory.next_event()? {
+        Some(event) => event,
         _ => bail!("Could not get event from test file"),
     };
 
@@ -61,7 +61,7 @@ pub(super) fn bench(ci: bool) -> Result<()> {
     let mut tracker = AddTracking::new();
     let mut series = EventSorter::new();
 
-    while let EventResult::Event(mut event) = factory.next_event(None)? {
+    while let Some(mut event) = factory.next_event()? {
         tracker.process_one(&mut event)?;
         series.add(event);
     }
diff --git a/retis/src/collect/collector.rs b/retis/src/collect/collector.rs
index 3cf36a8b5..1dc7d7807 100644
--- a/retis/src/collect/collector.rs
+++ b/retis/src/collect/collector.rs
@@ -18,7 +18,7 @@ use super::cli::Collect;
 use crate::{
     cli::{dynamic::DynamicCommand, CliConfig, CliDisplayFormat, FullCli, SubCommandRunner},
     core::{
-        events::{BpfEventsFactory, RetisEventsFactory},
+        events::{BpfEventsFactory, EventResult, RetisEventsFactory},
         filters::{
             filters::{BpfFilter, Filter},
             meta::filter::FilterMeta,
@@ -504,7 +504,6 @@ impl Collectors {
                         .iter_mut()
                         .try_for_each(|p| p.process_one(&event))?;
                 }
-                Eof => break,
                 Timeout => continue,
             }
         }
diff --git a/retis/src/core/events/bpf.rs b/retis/src/core/events/bpf.rs
index 8d7c15d69..963f6386b 100644
--- a/retis/src/core/events/bpf.rs
+++ b/retis/src/core/events/bpf.rs
@@ -64,6 +64,14 @@ macro_rules! event_byte_array {
     };
 }
 
+/// The return value of EventFactory::next_event()
+pub(crate) enum EventResult {
+    /// The Factory was able to create a new event.
+    Event(Event),
+    /// The timeout went off but a new attempt to retrieve an event might succeed.
+    Timeout,
+}
+
 /// BPF events factory retrieving and unmarshaling events coming from the BPF
 /// parts.
 #[cfg(not(test))]
diff --git a/retis/src/process/cli/pcap.rs b/retis/src/process/cli/pcap.rs
index 68d6e952a..16df3ba9d 100644
--- a/retis/src/process/cli/pcap.rs
+++ b/retis/src/process/cli/pcap.rs
@@ -284,11 +284,9 @@ where
 
     // See if we matched (not processed!) at least one event.
     let mut matched = false;
-
-    use EventResult::*;
     while run.running() {
-        match factory.next_event(Some(Duration::from_secs(1)))? {
-            Event(event) => {
+        match factory.next_event()? {
+            Some(event) => {
                 if let Some(kernel) = event.get_section::<KernelEvent>(SectionId::Kernel) {
                     // Check the event is matching the requested symbol.
                     if !filter(&kernel.probe_type, &kernel.symbol) {
@@ -299,8 +297,7 @@ where
                     parser.parse(&event)?;
                 }
             }
-            Eof => break,
-            Timeout => continue,
+            None => break,
         }
     }
 
diff --git a/retis/src/process/cli/print.rs b/retis/src/process/cli/print.rs
index 784955355..0550864fb 100644
--- a/retis/src/process/cli/print.rs
+++ b/retis/src/process/cli/print.rs
@@ -3,7 +3,7 @@
 //! Print is a simple post-processing command that just parses events and prints them back to
 //! stdout
 
-use std::{io::stdout, path::PathBuf, time::Duration};
+use std::{io::stdout, path::PathBuf};
 
 use anyhow::Result;
 use clap::Parser;
@@ -51,12 +51,10 @@ impl SubCommandParserRunner for Print {
         // Formatter & printer for events.
         let mut output = PrintEvent::new(Box::new(stdout()), PrintEventFormat::Text(format));
 
-        use EventResult::*;
         while run.running() {
-            match factory.next_event(Some(Duration::from_secs(1)))? {
-                Event(event) => output.process_one(&event)?,
-                Eof => break,
-                Timeout => continue,
+            match factory.next_event()? {
+                Some(event) => output.process_one(&event)?,
+                None => break,
             }
         }
         Ok(())
diff --git a/retis/src/process/cli/sort.rs b/retis/src/process/cli/sort.rs
index e7885006b..6adc9c3b0 100644
--- a/retis/src/process/cli/sort.rs
+++ b/retis/src/process/cli/sort.rs
@@ -118,8 +118,8 @@ impl SubCommandParserRunner for Sort {
         }
 
         while run.running() {
-            match factory.next_event(None)? {
-                EventResult::Event(mut event) => {
+            match factory.next_event()? {
+                Some(mut event) => {
                     // Add tracking information
                     tracker.process_one(&mut event)?;
 
@@ -139,8 +139,7 @@ impl SubCommandParserRunner for Sort {
                         }
                     }
                 }
-                EventResult::Eof => break,
-                EventResult::Timeout => continue,
+                None => break,
             }
         }
         // Flush remaining events

From 9e0038ef827e80aeaaddcdf7c52b6b63bf461c28 Mon Sep 17 00:00:00 2001
From: Antoine Tenart <atenart@redhat.com>
Date: Thu, 19 Sep 2024 21:37:59 +0200
Subject: [PATCH 08/16] python: add python event representation.

Use pyo3 to represent events and event sections.

Notably, this requires some simple enums to implement PartialEq.
Also, SectionId::from_str() has to be recovered.

EventTypes are given a __repr__() helper that uses the debug formatter.
It seems a good first start.

EventSections are given a __repr__() helper that prints a dictionary
based on the json object as well as a show() helper that formats the
event.

Co-authored-by: Adrian Moreno <amorenoz@redhat.com>
Signed-off-by: Antoine Tenart <atenart@redhat.com>
Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 Cargo.lock                       |  95 ++++++++++++++
 retis-derive/Cargo.toml          |   3 +
 retis-derive/src/lib.rs          | 101 +++++++++++++--
 retis-events/Cargo.toml          |   2 +
 retis-events/src/events.rs       |  37 +++++-
 retis-events/src/kernel.rs       |   7 ++
 retis-events/src/lib.rs          |   2 +
 retis-events/src/net.rs          |   9 ++
 retis-events/src/ovs.rs          |   2 +-
 retis-events/src/python.rs       | 206 +++++++++++++++++++++++++++++++
 retis-events/src/skb_tracking.rs |   1 +
 retis/Cargo.toml                 |   4 +-
 12 files changed, 454 insertions(+), 15 deletions(-)
 create mode 100644 retis-events/src/python.rs

diff --git a/Cargo.lock b/Cargo.lock
index c182ad5bc..2890b7c21 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -670,6 +670,18 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "indoc"
+version = "2.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5"
+
+[[package]]
+name = "inventory"
+version = "0.3.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767"
+
 [[package]]
 name = "is_terminal_polyfill"
 version = "1.70.0"
@@ -1098,6 +1110,12 @@ dependencies = [
  "pnet_macros_support",
 ]
 
+[[package]]
+name = "portable-atomic"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265"
+
 [[package]]
 name = "powerfmt"
 version = "0.2.0"
@@ -1145,6 +1163,70 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "pyo3"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "831e8e819a138c36e212f3af3fd9eeffed6bf1510a805af35b0edee5ffa59433"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "inventory",
+ "libc",
+ "memoffset 0.9.1",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e8730e591b14492a8945cdff32f089250b05f5accecf74aeddf9e8272ce1fa8"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e97e919d2df92eb88ca80a037969f44e5e70356559654962cbb3316d00300c6"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb57983022ad41f9e683a599f2fd13c3664d7063a3ac5714cae4b7bee7d3f206"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn 2.0.65",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec480c0c51ddec81019531705acac51bcdbeae563557c982aa8263bb96880372"
+dependencies = [
+ "heck 0.5.0",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn 2.0.65",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.36"
@@ -1266,6 +1348,7 @@ dependencies = [
  "chrono",
  "log",
  "once_cell",
+ "pyo3",
  "retis-derive",
  "serde",
  "serde_json",
@@ -1627,6 +1710,12 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "target-lexicon"
+version = "0.12.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+
 [[package]]
 name = "tempfile"
 version = "3.10.1"
@@ -1793,6 +1882,12 @@ version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
 
+[[package]]
+name = "unindent"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
+
 [[package]]
 name = "unsafe-libyaml"
 version = "0.2.11"
diff --git a/retis-derive/Cargo.toml b/retis-derive/Cargo.toml
index 22aebd820..7301c5cad 100644
--- a/retis-derive/Cargo.toml
+++ b/retis-derive/Cargo.toml
@@ -3,6 +3,9 @@ name = "retis-derive"
 version = "1.4.0"
 edition = "2021"
 
+[features]
+python = []
+
 [lib]
 proc-macro = true
 
diff --git a/retis-derive/src/lib.rs b/retis-derive/src/lib.rs
index 2dee16155..689ba6b38 100644
--- a/retis-derive/src/lib.rs
+++ b/retis-derive/src/lib.rs
@@ -1,5 +1,5 @@
 use quote::quote;
-use syn::{parse_macro_input, Item, ItemStruct};
+use syn::{parse_macro_input, Fields, Ident, Item, ItemStruct};
 
 #[proc_macro_attribute]
 pub fn raw_event_section(
@@ -55,26 +55,105 @@ pub fn event_section(
             {
                 serde_json::json!(self)
             }
+
+            #[cfg(feature = "python")]
+            fn to_py(&self, py: pyo3::Python<'_>) -> pyo3::PyObject {
+                use pyo3::IntoPy;
+                self.clone().into_py(py)
+            }
+        }
+
+        #[cfg_attr(feature = "python", pyo3::pymethods)]
+        #[cfg(feature = "python")]
+        impl #ident {
+            fn raw(&self, py: pyo3::Python<'_>) -> pyo3::PyObject {
+                crate::python::to_pyobject(&self.to_json(), py)
+            }
+
+            fn show(&self) -> String {
+                let format = crate::DisplayFormat::new().multiline(true);
+                format!("{}", self.display(&format, &crate::FormatterConf::new()))
+            }
         }
     };
     output.into()
 }
 
+struct EventTypeProps {
+    ident: Ident,
+    enum_is_simple: bool,
+    named_fields: bool,
+}
+
+fn item_get_props(item: &Item) -> EventTypeProps {
+    let mut enum_is_simple = false;
+    let named_fields;
+
+    let ident = match item {
+        Item::Struct(item) => {
+            named_fields = matches!(&item.fields, Fields::Named(_));
+            item.ident.clone()
+        }
+        Item::Enum(item) => {
+            named_fields = item
+                .variants
+                .iter()
+                .all(|v| matches!(v.fields, Fields::Named(_)));
+            enum_is_simple = item.variants.iter().all(|v| v.fields == Fields::Unit);
+            item.ident.clone()
+        }
+        _ => panic!("event types must be enums or structs"),
+    };
+    EventTypeProps {
+        ident,
+        enum_is_simple,
+        named_fields,
+    }
+}
+
 #[proc_macro_attribute]
 pub fn event_type(
-    _: proc_macro::TokenStream,
+    _args: proc_macro::TokenStream,
     item: proc_macro::TokenStream,
 ) -> proc_macro::TokenStream {
-    let output = format!(
-        r#"
+    let input: Item = parse_macro_input!(item);
+    let props = item_get_props(&input);
+    let mut pyclass_args = Vec::new();
+    let mut derives = vec![
+        quote!(Clone),
+        quote!(Debug),
+        quote!(serde::Serialize),
+        quote!(serde::Deserialize),
+    ];
+
+    if props.enum_is_simple {
+        // Simple enums need to be passed extra arguments so equality is implemented
+        // using underlying integers: See https://pyo3.rs/main/doc/pyo3/attr.pyclass.html.
+        pyclass_args.push(quote!(eq));
+        pyclass_args.push(quote!(eq_int));
+        derives.push(quote!(PartialEq))
+    } else if props.named_fields {
+        // Generate getters to all named fields.
+        pyclass_args.push(quote!(get_all));
+    }
+    let ident = &props.ident;
+
+    let output = quote! {
+        #[cfg_attr(feature = "python", pyo3::pyclass(#(#pyclass_args),*))]
         #[serde_with::skip_serializing_none]
-        #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
-        {item}
-    "#
-    );
-    output
-        .parse()
-        .expect("Invalid tokens from event_section macro")
+        #[derive(#(#derives),*)]
+        #input
+
+        #[cfg_attr(feature = "python", pyo3::pymethods)]
+        #[cfg(feature = "python")]
+        impl #ident {
+            fn __repr__(&self, py: pyo3::Python<'_>) -> String {
+                format!("{:?}", self)
+            }
+
+        }
+    };
+    output.into()
 }
 
 #[proc_macro_attribute]
diff --git a/retis-events/Cargo.toml b/retis-events/Cargo.toml
index c9505e83a..f6a96ed5a 100644
--- a/retis-events/Cargo.toml
+++ b/retis-events/Cargo.toml
@@ -4,6 +4,7 @@ version = "1.4.0"
 edition = "2021"
 
 [features]
+python = ["dep:pyo3", "retis-derive/python"]
 test-events = []
 
 [lib]
@@ -16,6 +17,7 @@ chrono = "0.4"
 log = { version = "0.4", features = ["std"] }
 once_cell = "1.15"
 retis-derive = {version = "1.4", path = "../retis-derive"}
+pyo3 = {version = "0.22", features = ["gil-refs", "auto-initialize", "multiple-pymethods"], optional = true}
 serde = {version = "1.0", features = ["derive"]}
 serde_json = "1.0"
 serde_with = "3.0"
diff --git a/retis-events/src/events.rs b/retis-events/src/events.rs
index e197876c5..c13cc1130 100644
--- a/retis-events/src/events.rs
+++ b/retis-events/src/events.rs
@@ -34,7 +34,7 @@
 #![allow(dead_code)] // FIXME
 #![allow(clippy::wrong_self_convention)]
 
-use std::{any::Any, collections::HashMap, fmt};
+use std::{any::Any, collections::HashMap, fmt, str::FromStr};
 
 use anyhow::{anyhow, bail, Result};
 use log::debug;
@@ -106,6 +106,11 @@ impl Event {
         }
     }
 
+    #[allow(clippy::borrowed_box)]
+    pub(super) fn get(&self, owner: SectionId) -> Option<&Box<dyn EventSection>> {
+        self.0.get(&owner)
+    }
+
     pub fn to_json(&self) -> serde_json::Value {
         let mut event = serde_json::Map::new();
 
@@ -242,6 +247,29 @@ impl fmt::Display for SectionId {
     }
 }
 
+impl FromStr for SectionId {
+    type Err = anyhow::Error;
+
+    /// Constructs an SectionId from a section unique str identifier.
+    fn from_str(val: &str) -> Result<Self> {
+        use SectionId::*;
+        Ok(match val {
+            "common" => Common,
+            "kernel" => Kernel,
+            "userspace" => Userspace,
+            "tracking" => Tracking,
+            "skb-tracking" => SkbTracking,
+            "skb-drop" => SkbDrop,
+            "skb" => Skb,
+            "ovs" => Ovs,
+            "nft" => Nft,
+            "ct" => Ct,
+            "startup" => Startup,
+            x => bail!("Can't construct a SectionId from {}", x),
+        })
+    }
+}
+
 type EventSectionMap = HashMap<String, fn(serde_json::Value) -> Result<Box<dyn EventSection>>>;
 static EVENT_SECTIONS: OnceCell<EventSectionMap> = OnceCell::new();
 
@@ -297,6 +325,8 @@ pub trait EventSectionInternal {
     fn as_any(&self) -> &dyn Any;
     fn as_any_mut(&mut self) -> &mut dyn Any;
     fn to_json(&self) -> serde_json::Value;
+    #[cfg(feature = "python")]
+    fn to_py(&self, py: pyo3::Python<'_>) -> pyo3::PyObject;
 }
 
 // We need this as the value given as the input when deserializing something
@@ -317,6 +347,11 @@ impl EventSectionInternal for () {
     fn to_json(&self) -> serde_json::Value {
         serde_json::Value::Null
     }
+
+    #[cfg(feature = "python")]
+    fn to_py(&self, py: pyo3::Python<'_>) -> pyo3::PyObject {
+        py.None()
+    }
 }
 
 /// A set of sorted Events with the same tracking id.
diff --git a/retis-events/src/kernel.rs b/retis-events/src/kernel.rs
index 965621f34..fee2cff2d 100644
--- a/retis-events/src/kernel.rs
+++ b/retis-events/src/kernel.rs
@@ -58,3 +58,10 @@ impl EventFmt for StackTrace {
         }
     }
 }
+
+#[cfg(feature = "python")]
+impl pyo3::ToPyObject for StackTrace {
+    fn to_object(&self, py: pyo3::Python<'_>) -> pyo3::PyObject {
+        pyo3::IntoPy::into_py(self.0.clone(), py)
+    }
+}
diff --git a/retis-events/src/lib.rs b/retis-events/src/lib.rs
index c754dd647..6bcdda709 100644
--- a/retis-events/src/lib.rs
+++ b/retis-events/src/lib.rs
@@ -14,6 +14,8 @@ pub use display::*;
 
 pub mod file;
 pub mod net;
+#[cfg(feature = "python")]
+pub mod python;
 
 pub mod common;
 pub use common::*;
diff --git a/retis-events/src/net.rs b/retis-events/src/net.rs
index 14811f2d6..47f622d1b 100644
--- a/retis-events/src/net.rs
+++ b/retis-events/src/net.rs
@@ -53,6 +53,8 @@ pub(crate) fn protocol_str(protocol: u8) -> Option<&'static str> {
 }
 
 /// Represents a raw packet. Stored internally as a `Vec<u8>`.
+/// We don't use #[event_type] as we're implementing serde::Serialize and
+/// serde::Deserialize manually.
 #[derive(Clone, Debug)]
 pub struct RawPacket(pub Vec<u8>);
 
@@ -96,3 +98,10 @@ impl<'de> serde::Deserialize<'de> for RawPacket {
         deserializer.deserialize_str(RawPacketVisitor)
     }
 }
+
+#[cfg(feature = "python")]
+impl pyo3::ToPyObject for RawPacket {
+    fn to_object(&self, py: pyo3::Python<'_>) -> pyo3::PyObject {
+        pyo3::IntoPy::into_py(self.0.clone(), py)
+    }
+}
diff --git a/retis-events/src/ovs.rs b/retis-events/src/ovs.rs
index 1ea3d5a1a..e674a8530 100644
--- a/retis-events/src/ovs.rs
+++ b/retis-events/src/ovs.rs
@@ -504,7 +504,7 @@ impl OvsActionCt {
 }
 
 #[event_type]
-#[derive(Default, PartialEq)]
+#[derive(Default)]
 pub enum NatDirection {
     #[default]
     #[serde(rename = "src")]
diff --git a/retis-events/src/python.rs b/retis-events/src/python.rs
new file mode 100644
index 000000000..4cb4483ca
--- /dev/null
+++ b/retis-events/src/python.rs
@@ -0,0 +1,206 @@
+//! # Python bindings
+//!
+//! This module contains python bindings for retis events so that they can
+//! be inspected in post-processing tools written in python.
+
+use std::{collections::HashMap, path::PathBuf, str::FromStr};
+
+use pyo3::{
+    exceptions::{PyKeyError, PyRuntimeError},
+    prelude::*,
+};
+
+use super::*;
+
+/// Python representation of an Event.
+///
+/// This class exposes the event data as well as some helper functions
+/// to inspect an event.
+///
+/// # Accessing event data
+///
+/// The Event class allows accessing each section by using the __getitem__
+/// function. The object returned is of a builtin type whose attributes can
+/// be accessed directly.
+///
+/// In addition, some helpers might be available. One of the helpers that
+/// is implemented for all event section types is `raw()`, which returns
+/// the data as a dictionary.
+///
+/// ## Examples
+///
+/// ```text
+/// >>> print(event["skb"])
+/// {'tcp': {'sport': 35082, 'window': 9285, 'ack_seq': 3083383182, 'doff': 8, 'dport': 8080, 'flags': 24, 'seq': 132765809}, 'ip': {'ttl': 64, 'v4': {'tos': 0, 'offset': 0, 'id': 53289, 'flags': 2}, 'ecn': 0, 'len': 91, 'protocol': 6, 'daddr': '127.0.0.1', 'saddr': '127.0.0.1'}, 'dev': {'ifindex': 1, 'name': 'lo'}}
+///
+/// >>> print(event["skb"].tcp.dport)
+/// 8080
+/// ```
+///
+/// # Displaying events
+///
+/// Another helper implemented for all event types as well as for the Event
+/// class is `show()` which returns a string representation of the event, similar
+/// to how `retis print` would print it.
+///
+/// ## Examples
+///
+/// ```text
+/// >>> print(e.show())
+/// 633902702662502 (8) [scapy] 2856768 [tp] net:net_dev_queue #24087f96a1366ffff8fa9b9718500 (skb ffff8fa94fabd500)
+///   if 15 (p1_p) 2001:db8:dead::1.20 > 2001:db8:dead::2.80 ttl 64 len 20 proto TCP (6) flags [S] seq 0 win 8192
+/// ```
+#[pyclass(name = "Event")]
+pub struct PyEvent(Event);
+
+impl PyEvent {
+    pub(crate) fn new(event: Event) -> Self {
+        Self(event)
+    }
+}
+
+#[pymethods]
+impl PyEvent {
+    /// Controls how the PyEvent is represented, eg. what is the output of
+    /// `print(e)`.
+    fn __repr__<'a>(&'a self, py: Python<'a>) -> String {
+        let raw = self.raw(py);
+        let dict: &Bound<'_, PyAny> = raw.bind(py);
+        dict.repr().unwrap().to_string()
+    }
+
+    /// Allows to use the object as a dictionary, eg. `e['skb']`.
+    fn __getitem__<'a>(&'a self, py: Python<'a>, attr: &str) -> PyResult<Py<PyAny>> {
+        if let Ok(id) = SectionId::from_str(attr) {
+            if let Some(section) = self.0.get(id) {
+                return Ok(section.to_py(py));
+            }
+        }
+        Err(PyKeyError::new_err(attr.to_string()))
+    }
+
+    /// Allows to check if a section is present inthe event, e.g: `'skb' in e`
+    fn __contains__<'a>(&'a self, _py: Python<'a>, attr: &str) -> PyResult<bool> {
+        if let Ok(id) = SectionId::from_str(attr) {
+            if self.0.get(id).is_some() {
+                return Ok(true);
+            }
+        }
+        Ok(false)
+    }
+
+    /// Returns internal data as a dictionary
+    ///
+    /// Returns a dictionary with all key<>data stored (recursively) in the
+    /// event, eg. `e.raw()['skb']['dev']`.
+    fn raw(&self, py: Python<'_>) -> PyObject {
+        to_pyobject(&self.0.to_json(), py)
+    }
+
+    /// Returns a string representation of the event
+    fn show(&self) -> String {
+        let format = crate::DisplayFormat::new().multiline(true);
+        format!("{}", self.0.display(&format, &crate::FormatterConf::new()))
+    }
+}
+
+/// Python event reader
+///
+/// Objects of this class can read events from unsorted event files.
+///
+/// ## Example
+///
+/// ```python
+/// reader = EventReader("retis.data")
+/// for event in reader:
+///     print(event.show())
+/// ```
+#[pyclass(name = "EventReader")]
+pub(crate) struct PyEventReader {
+    factory: file::FileEventsFactory,
+}
+
+#[pymethods]
+impl PyEventReader {
+    #[new]
+    pub(crate) fn new(path: PathBuf) -> PyResult<Self> {
+        let factory = file::FileEventsFactory::new(path)
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+        Ok(PyEventReader { factory })
+    }
+
+    // Implementation of the iterator protocol.
+    pub(crate) fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
+        slf
+    }
+
+    pub(crate) fn __next__(
+        mut slf: PyRefMut<'_, Self>,
+        py: Python<'_>,
+    ) -> PyResult<Option<Py<PyAny>>> {
+        match slf
+            .factory
+            .next_event()
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?
+        {
+            Some(event) => {
+                let pyevent: Bound<'_, PyEvent> = Bound::new(py, PyEvent::new(event))?;
+                Ok(Some(pyevent.into_any().into()))
+            }
+            None => Ok(None),
+        }
+    }
+}
+
+/// Python event file
+///
+/// Objects of this class can read files generated by retis and create
+/// EventReader instances to iterate over their content.
+///
+/// ## Example
+///
+/// ```python
+/// event_file = EventFile("retis.data")
+/// for event in event_file.events():
+///     print(event.show())
+/// ```
+#[pyclass(name = "EventFile")]
+pub(crate) struct PyEventFile {
+    path: PathBuf,
+}
+
+#[pymethods]
+impl PyEventFile {
+    #[new]
+    pub(crate) fn new(path: PathBuf) -> PyResult<Self> {
+        Ok(PyEventFile { path })
+    }
+
+    pub(crate) fn events(&self) -> PyResult<PyEventReader> {
+        PyEventReader::new(self.path.clone())
+    }
+}
+
+/// Converts a serde_json::Value to a PyObject.
+pub(crate) fn to_pyobject(val: &serde_json::Value, py: Python<'_>) -> PyObject {
+    use serde_json::Value;
+    match val {
+        Value::Null => py.None(),
+        Value::Bool(b) => b.to_object(py),
+        Value::Number(n) => n
+            .as_i64()
+            .map(|x| x.to_object(py))
+            .or(n.as_u64().map(|x| x.to_object(py)))
+            .or(n.as_f64().map(|x| x.to_object(py)))
+            .expect("Cannot convert number to Python object"),
+        Value::String(s) => s.to_object(py),
+        Value::Array(a) => {
+            let vec: Vec<_> = a.iter().map(|x| to_pyobject(x, py)).collect();
+            vec.to_object(py)
+        }
+        Value::Object(o) => {
+            let map: HashMap<_, _> = o.iter().map(|(k, v)| (k, to_pyobject(v, py))).collect();
+            map.to_object(py)
+        }
+    }
+}
diff --git a/retis-events/src/skb_tracking.rs b/retis-events/src/skb_tracking.rs
index 7937f5cdc..a39dcf76b 100644
--- a/retis-events/src/skb_tracking.rs
+++ b/retis-events/src/skb_tracking.rs
@@ -28,6 +28,7 @@ pub struct SkbTrackingEvent {
 }
 
 #[allow(dead_code)]
+#[cfg_attr(feature = "python", pyo3::pymethods)]
 impl SkbTrackingEvent {
     /// Get the tracking id.
     pub fn tracking_id(&self) -> u128 {
diff --git a/retis/Cargo.toml b/retis/Cargo.toml
index 76ef2fbd9..ab54c19fd 100644
--- a/retis/Cargo.toml
+++ b/retis/Cargo.toml
@@ -33,7 +33,7 @@ caps = "0.5"
 clap = { version = "4.0", features = ["derive", "string"] }
 clap_complete = "4.4"
 elf = "0.7"
-events = {version = "1.4", path = "../retis-events", package="retis-events"}
+events = {version = "1.4", path = "../retis-events", package="retis-events", features=["python"]}
 flate2 = "1.0"
 libbpf-rs = "0.22"
 libbpf-sys = "=1.2"
@@ -49,7 +49,7 @@ plain = "0.2"
 pnet_packet = "0.34"
 rbpf = {version = "0.2", optional = true}
 regex = "1.7"
-retis-derive = {version = "1.4", path = "../retis-derive"}
+retis-derive = {version = "1.4", path = "../retis-derive", features=["python"]}
 serde = {version = "1.0", features = ["derive"]}
 serde_json = "1.0"
 serde_with = "3.0"

From c851aef2eccadb7742b7d060e62d6846f054025f Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Tue, 17 Sep 2024 18:11:28 +0200
Subject: [PATCH 09/16] events: add interactive python shell

Add an python shell as post-process command.

Signed-off-by: Antoine Tenart <atenart@redhat.com>
Co-authored-by: Antoine Tenart <atenart@redhat.com>
Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 Vagrantfile                     |  2 ++
 retis-events/src/python.rs      | 45 ++++++++++++++++++++++++++++++++-
 retis/src/cli/cli.rs            |  1 +
 retis/src/process/cli/mod.rs    |  3 +++
 retis/src/process/cli/python.rs | 27 ++++++++++++++++++++
 5 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 retis/src/process/cli/python.rs

diff --git a/Vagrantfile b/Vagrantfile
index 7737a9b9e..d32709b72 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -15,6 +15,7 @@ dnf install -y \
     libpcap-devel \
     git \
     python3-pip \
+    python3-devel \
     socat \
     nftables \
     make \
@@ -111,6 +112,7 @@ Vagrant.configure("2") do |config|
           git \
           pkg-config \
           python3-pip \
+          python3-dev \
           openvswitch-switch \
           socat \
           nftables \
diff --git a/retis-events/src/python.rs b/retis-events/src/python.rs
index 4cb4483ca..561d3816e 100644
--- a/retis-events/src/python.rs
+++ b/retis-events/src/python.rs
@@ -3,11 +3,13 @@
 //! This module contains python bindings for retis events so that they can
 //! be inspected in post-processing tools written in python.
 
-use std::{collections::HashMap, path::PathBuf, str::FromStr};
+use std::{collections::HashMap, fs, path::PathBuf, str::FromStr};
 
+use anyhow::Result;
 use pyo3::{
     exceptions::{PyKeyError, PyRuntimeError},
     prelude::*,
+    types::PyDict,
 };
 
 use super::*;
@@ -204,3 +206,44 @@ pub(crate) fn to_pyobject(val: &serde_json::Value, py: Python<'_>) -> PyObject {
         }
     }
 }
+
+/// Create a python shell and execute the provided script.
+pub fn shell_execute(file: PathBuf, script: Option<&PathBuf>) -> Result<()> {
+    let event_file = PyEventFile::new(file)?;
+
+    Python::with_gil(|py| -> PyResult<()> {
+        let shell = PyShell::new(py, event_file)?;
+        if let Some(script) = script {
+            shell.run(&fs::read_to_string(script)?)
+        } else {
+            shell.interact()
+        }
+    })?;
+    Ok(())
+}
+
+/// Python shell.
+struct PyShell<'a> {
+    py: Python<'a>,
+    globals: Bound<'a, PyDict>,
+}
+
+impl<'a> PyShell<'a> {
+    const INTERACTIVE_SHELL: &'static str = "import code; code.interact(local=locals())";
+
+    fn new(py: Python<'a>, file: PyEventFile) -> PyResult<Self> {
+        let globals = PyDict::new_bound(py);
+        globals.set_item("reader", Py::new(py, file)?.into_bound(py))?;
+
+        Ok(Self { py, globals })
+    }
+
+    fn run(&self, script: &str) -> PyResult<()> {
+        self.py
+            .run_bound(script, Some(&self.globals.as_borrowed()), None)
+    }
+
+    fn interact(&self) -> PyResult<()> {
+        self.run(Self::INTERACTIVE_SHELL)
+    }
+}
diff --git a/retis/src/cli/cli.rs b/retis/src/cli/cli.rs
index 8dee7cead..0ad3bfa08 100644
--- a/retis/src/cli/cli.rs
+++ b/retis/src/cli/cli.rs
@@ -432,6 +432,7 @@ pub(crate) fn get_cli() -> Result<ThinCli> {
     cli.add_subcommand(Box::new(Collect::new()?))?;
     cli.add_subcommand(Box::new(Print::new()?))?;
     cli.add_subcommand(Box::new(Sort::new()?))?;
+    cli.add_subcommand(Box::new(PythonCli::new()?))?;
     cli.add_subcommand(Box::new(Pcap::new()?))?;
     cli.add_subcommand(Box::new(Inspect::new()?))?;
     cli.add_subcommand(Box::new(ProfileCmd::new()?))?;
diff --git a/retis/src/process/cli/mod.rs b/retis/src/process/cli/mod.rs
index af8778a4b..1d33bd721 100644
--- a/retis/src/process/cli/mod.rs
+++ b/retis/src/process/cli/mod.rs
@@ -8,5 +8,8 @@ pub(crate) use self::pcap::*;
 pub(crate) mod print;
 pub(crate) use print::*;
 
+pub(crate) mod python;
+pub(crate) use python::*;
+
 pub(crate) mod sort;
 pub(crate) use sort::*;
diff --git a/retis/src/process/cli/python.rs b/retis/src/process/cli/python.rs
new file mode 100644
index 000000000..68924e87f
--- /dev/null
+++ b/retis/src/process/cli/python.rs
@@ -0,0 +1,27 @@
+use std::path::PathBuf;
+
+use anyhow::Result;
+use clap::{arg, Parser};
+
+use crate::{cli::*, events::python::shell_execute, module::Modules};
+
+/// Runs Python scripts with events imported.
+#[derive(Parser, Debug, Default)]
+#[command(name = "python")]
+pub(crate) struct PythonCli {
+    #[arg(
+        long,
+        short,
+        default_value = "retis.data",
+        help = "File from which to read events"
+    )]
+    pub(super) input: PathBuf,
+    #[arg(help = "Python script to execute. Omit to drop into an interactive shell.")]
+    pub(super) script: Option<PathBuf>,
+}
+
+impl SubCommandParserRunner for PythonCli {
+    fn run(&mut self, _modules: Modules) -> Result<()> {
+        shell_execute(self.input.clone(), self.script.as_ref())
+    }
+}

From 3395af4b0f3349f386def6f2a67e5d0c3fe86568 Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Tue, 20 Aug 2024 17:28:16 +0200
Subject: [PATCH 10/16] events: add section iterator.

Add a helper to iterate through the available sections of an event.
Expose such helper to the python representation.

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 Cargo.lock                 | 440 +++++++++++++++++++------------------
 retis-events/src/events.rs |   5 +
 retis-events/src/python.rs |  10 +-
 3 files changed, 243 insertions(+), 212 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2890b7c21..d1f7fbbb1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3,10 +3,10 @@
 version = 3
 
 [[package]]
-name = "adler"
-version = "1.0.2"
+name = "adler2"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
 
 [[package]]
 name = "aho-corasick"
@@ -34,9 +34,9 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.6.14"
+version = "0.6.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b"
+checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
 dependencies = [
  "anstyle",
  "anstyle-parse",
@@ -49,33 +49,33 @@ dependencies = [
 
 [[package]]
 name = "anstyle"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
+checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.4"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"
+checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
 dependencies = [
  "utf8parse",
 ]
 
 [[package]]
 name = "anstyle-query"
-version = "1.0.3"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5"
+checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
 dependencies = [
  "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "anstyle-wincon"
-version = "3.0.3"
+version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19"
+checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
 dependencies = [
  "anstyle",
  "windows-sys 0.52.0",
@@ -83,9 +83,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.86"
+version = "1.0.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
+checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6"
 
 [[package]]
 name = "ascii"
@@ -95,9 +95,9 @@ checksum = "3ae7d751998c189c1d4468cf0a39bb2eae052a9c58d50ebb3b9591ee3813ad50"
 
 [[package]]
 name = "autocfg"
-version = "1.3.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
+checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
 
 [[package]]
 name = "base-x"
@@ -117,7 +117,7 @@ version = "0.69.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cexpr",
  "clang-sys",
  "itertools",
@@ -130,7 +130,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.65",
+ "syn 2.0.79",
  "which",
 ]
 
@@ -142,15 +142,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.5.0"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
+checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
 
 [[package]]
 name = "btf-rs"
-version = "1.1.0"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1e4d11d3ab5f700c880746603ae231e62dcfdfd29a3a0977f6022a35fc50285"
+checksum = "fe86bb5e4b3b6428b947ecfd9bef6c504c000c0174200dc52a2df432d5fd88fc"
 dependencies = [
  "anyhow",
  "byteorder",
@@ -179,9 +179,9 @@ dependencies = [
 
 [[package]]
 name = "camino"
-version = "1.1.7"
+version = "1.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239"
+checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3"
 dependencies = [
  "serde",
 ]
@@ -221,9 +221,12 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.98"
+version = "1.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f"
+checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938"
+dependencies = [
+ "shlex",
+]
 
 [[package]]
 name = "cexpr"
@@ -263,20 +266,20 @@ dependencies = [
 
 [[package]]
 name = "clang-sys"
-version = "1.7.0"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1"
+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
 dependencies = [
  "glob",
  "libc",
- "libloading 0.8.3",
+ "libloading 0.8.5",
 ]
 
 [[package]]
 name = "clap"
-version = "4.5.4"
+version = "4.5.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0"
+checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -284,9 +287,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.2"
+version = "4.5.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4"
+checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b"
 dependencies = [
  "anstream",
  "anstyle",
@@ -296,36 +299,36 @@ dependencies = [
 
 [[package]]
 name = "clap_complete"
-version = "4.5.2"
+version = "4.5.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd79504325bf38b10165b02e89b4347300f855f273c4cb30c4a3209e6583275e"
+checksum = "8937760c3f4c60871870b8c3ee5f9b30771f792a7045c48bcbba999d7d6b3b8e"
 dependencies = [
  "clap",
 ]
 
 [[package]]
 name = "clap_derive"
-version = "4.5.4"
+version = "4.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64"
+checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "clap_lex"
-version = "0.7.0"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
+checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
 
 [[package]]
 name = "colorchoice"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
+checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
 
 [[package]]
 name = "combine"
@@ -345,9 +348,9 @@ checksum = "373e9fafaa20882876db20562275ff58d50e0caa2590077fe7ce7bef90211d0d"
 
 [[package]]
 name = "core-foundation-sys"
-version = "0.8.6"
+version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
 name = "crc32fast"
@@ -360,9 +363,9 @@ dependencies = [
 
 [[package]]
 name = "darling"
-version = "0.20.9"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1"
+checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
 dependencies = [
  "darling_core",
  "darling_macro",
@@ -370,27 +373,27 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.20.9"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120"
+checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
 dependencies = [
  "fnv",
  "ident_case",
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "darling_macro"
-version = "0.20.9"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178"
+checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
 dependencies = [
  "darling_core",
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -422,9 +425,9 @@ checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0"
 
 [[package]]
 name = "either"
-version = "1.12.0"
+version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b"
+checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
 
 [[package]]
 name = "elf"
@@ -471,15 +474,15 @@ dependencies = [
 
 [[package]]
 name = "fastrand"
-version = "2.1.0"
+version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
+checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
 
 [[package]]
 name = "flate2"
-version = "1.0.30"
+version = "1.0.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae"
+checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0"
 dependencies = [
  "crc32fast",
  "miniz_oxide",
@@ -582,9 +585,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
 [[package]]
 name = "hashbrown"
-version = "0.14.5"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb"
 
 [[package]]
 name = "heck"
@@ -621,9 +624,9 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.60"
+version = "0.1.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141"
+checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
@@ -661,12 +664,12 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.2.6"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
+checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da"
 dependencies = [
  "equivalent",
- "hashbrown 0.14.5",
+ "hashbrown 0.15.0",
  "serde",
 ]
 
@@ -684,9 +687,9 @@ checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767"
 
 [[package]]
 name = "is_terminal_polyfill"
-version = "1.70.0"
+version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
 
 [[package]]
 name = "itertools"
@@ -705,18 +708,18 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
 
 [[package]]
 name = "js-sys"
-version = "0.3.69"
+version = "0.3.70"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
+checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a"
 dependencies = [
  "wasm-bindgen",
 ]
 
 [[package]]
 name = "lazy_static"
-version = "1.4.0"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
 [[package]]
 name = "lazycell"
@@ -752,7 +755,7 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "98e789d5b4dab748a2b8415974bf5b0fc2e9e782d0b118166222195acb4ac3b6"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "lazy_static",
  "libbpf-sys",
  "libc",
@@ -777,9 +780,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.155"
+version = "0.2.159"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
 
 [[package]]
 name = "libloading"
@@ -793,9 +796,9 @@ dependencies = [
 
 [[package]]
 name = "libloading"
-version = "0.8.3"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19"
+checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
 dependencies = [
  "cfg-if",
  "windows-targets",
@@ -819,15 +822,15 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.21"
+version = "0.4.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
+checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
 
 [[package]]
 name = "memchr"
-version = "2.7.2"
+version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
 [[package]]
 name = "memmap2"
@@ -840,9 +843,9 @@ dependencies = [
 
 [[package]]
 name = "memmap2"
-version = "0.9.4"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322"
+checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
 dependencies = [
  "libc",
 ]
@@ -873,11 +876,11 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
 [[package]]
 name = "miniz_oxide"
-version = "0.7.3"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae"
+checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
 dependencies = [
- "adler",
+ "adler2",
 ]
 
 [[package]]
@@ -899,7 +902,7 @@ version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cfg-if",
  "libc",
  "memoffset 0.9.1",
@@ -911,7 +914,7 @@ version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cfg-if",
  "cfg_aliases",
  "libc",
@@ -981,9 +984,12 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.19.0"
+version = "1.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1"
+dependencies = [
+ "portable-atomic",
+]
 
 [[package]]
 name = "pager"
@@ -997,9 +1003,9 @@ dependencies = [
 
 [[package]]
 name = "parking_lot"
-version = "0.12.2"
+version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb"
+checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
 dependencies = [
  "lock_api",
  "parking_lot_core",
@@ -1058,9 +1064,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pkg-config"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
+checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
 
 [[package]]
 name = "plain"
@@ -1086,7 +1092,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -1112,9 +1118,9 @@ dependencies = [
 
 [[package]]
 name = "portable-atomic"
-version = "1.7.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265"
+checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2"
 
 [[package]]
 name = "powerfmt"
@@ -1124,12 +1130,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
 [[package]]
 name = "prettyplease"
-version = "0.2.20"
+version = "0.2.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e"
+checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba"
 dependencies = [
  "proc-macro2",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -1156,18 +1162,18 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.83"
+version = "1.0.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b33eb56c327dec362a9e55b3ad14f9d2f0904fb5a5b03b513ab5465399e9f43"
+checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "pyo3"
-version = "0.22.2"
+version = "0.22.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "831e8e819a138c36e212f3af3fd9eeffed6bf1510a805af35b0edee5ffa59433"
+checksum = "15ee168e30649f7f234c3d49ef5a7a6cbf5134289bc46c29ff3155fa3221c225"
 dependencies = [
  "cfg-if",
  "indoc",
@@ -1184,9 +1190,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.22.2"
+version = "0.22.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e8730e591b14492a8945cdff32f089250b05f5accecf74aeddf9e8272ce1fa8"
+checksum = "e61cef80755fe9e46bb8a0b8f20752ca7676dcc07a5277d8b7768c6172e529b3"
 dependencies = [
  "once_cell",
  "target-lexicon",
@@ -1194,9 +1200,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.22.2"
+version = "0.22.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e97e919d2df92eb88ca80a037969f44e5e70356559654962cbb3316d00300c6"
+checksum = "67ce096073ec5405f5ee2b8b31f03a68e02aa10d5d4f565eca04acc41931fa1c"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -1204,34 +1210,34 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.22.2"
+version = "0.22.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb57983022ad41f9e683a599f2fd13c3664d7063a3ac5714cae4b7bee7d3f206"
+checksum = "2440c6d12bc8f3ae39f1e775266fa5122fd0c8891ce7520fa6048e683ad3de28"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.22.2"
+version = "0.22.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec480c0c51ddec81019531705acac51bcdbeae563557c982aa8263bb96880372"
+checksum = "1be962f0e06da8f8465729ea2cb71a416d2257dff56cbe40a70d3e62a93ae5d1"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "pyo3-build-config",
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.36"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
 dependencies = [
  "proc-macro2",
 ]
@@ -1250,18 +1256,18 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.1"
+version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e"
+checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
 ]
 
 [[package]]
 name = "regex"
-version = "1.10.4"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
+checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1271,9 +1277,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.6"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
+checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1282,9 +1288,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.8.3"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56"
+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
 [[package]]
 name = "retis"
@@ -1305,7 +1311,7 @@ dependencies = [
  "libbpf-sys",
  "libc",
  "log",
- "memmap2 0.9.4",
+ "memmap2 0.9.5",
  "memoffset 0.9.1",
  "nix 0.28.0",
  "once_cell",
@@ -1336,7 +1342,7 @@ name = "retis-derive"
 version = "1.4.0"
 dependencies = [
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -1372,11 +1378,11 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.38.34"
+version = "0.38.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
+checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "errno 0.3.9",
  "libc",
  "linux-raw-sys",
@@ -1397,9 +1403,9 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
 
 [[package]]
 name = "scc"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76ad2bbb0ae5100a07b7a6f2ed7ab5fd0045551a4c507989b7a620046ea3efdc"
+checksum = "836f1e0f4963ef5288b539b643b35e043e76a32d0f4e47e67febf69576527f50"
 dependencies = [
  "sdd",
 ]
@@ -1424,14 +1430,14 @@ checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "sdd"
-version = "0.2.0"
+version = "3.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b84345e4c9bd703274a082fb80caaa99b7612be48dfaa1dd9266577ec412309d"
+checksum = "60a7b59a5d9b0099720b417b6325d91a52cbf5b3dcb5041d864be53eefa58abc"
 
 [[package]]
 name = "semver"
@@ -1459,46 +1465,47 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
 
 [[package]]
 name = "serde"
-version = "1.0.202"
+version = "1.0.210"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395"
+checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.202"
+version = "1.0.210"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838"
+checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.117"
+version = "1.0.128"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3"
+checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
 dependencies = [
  "itoa",
+ "memchr",
  "ryu",
  "serde",
 ]
 
 [[package]]
 name = "serde_with"
-version = "3.8.1"
+version = "3.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20"
+checksum = "9720086b3357bcb44fce40117d769a4d068c70ecfa190850a980a71755f66fcc"
 dependencies = [
  "base64",
  "chrono",
  "hex",
  "indexmap 1.9.3",
- "indexmap 2.2.6",
+ "indexmap 2.6.0",
  "serde",
  "serde_derive",
  "serde_json",
@@ -1508,14 +1515,14 @@ dependencies = [
 
 [[package]]
 name = "serde_with_macros"
-version = "3.8.1"
+version = "3.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2"
+checksum = "5f1abbfe725f27678f4663bcacb75a83e829fd464c25d78dd038a3a29e307cec"
 dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -1524,7 +1531,7 @@ version = "0.9.34+deprecated"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
 dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.6.0",
  "itoa",
  "ryu",
  "serde",
@@ -1553,7 +1560,7 @@ checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -1567,9 +1574,9 @@ dependencies = [
 
 [[package]]
 name = "sha1_smol"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
+checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"
 
 [[package]]
 name = "shlex"
@@ -1701,9 +1708,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.65"
+version = "2.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2863d96a84c6439701d7a38f9de935ec562c8832cc55d1dde0f513b52fad106"
+checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1718,14 +1725,15 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
 
 [[package]]
 name = "tempfile"
-version = "3.10.1"
+version = "3.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
+checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b"
 dependencies = [
  "cfg-if",
  "fastrand",
+ "once_cell",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -1755,7 +1763,7 @@ dependencies = [
  "cfg-if",
  "proc-macro2",
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -1766,28 +1774,28 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
  "test-case-core",
 ]
 
 [[package]]
 name = "thiserror"
-version = "1.0.61"
+version = "1.0.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709"
+checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.61"
+version = "1.0.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533"
+checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -1861,9 +1869,9 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "0.6.6"
+version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf"
+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
 
 [[package]]
 name = "toml_edit"
@@ -1871,16 +1879,16 @@ version = "0.19.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
 dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.6.0",
  "toml_datetime",
  "winnow",
 ]
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.12"
+version = "1.0.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
 
 [[package]]
 name = "unindent"
@@ -1896,15 +1904,15 @@ checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
 
 [[package]]
 name = "utf8parse"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "version_check"
-version = "0.9.4"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
 [[package]]
 name = "vsprintf"
@@ -1918,34 +1926,35 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
+checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
 dependencies = [
  "cfg-if",
+ "once_cell",
  "wasm-bindgen-macro",
 ]
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
+checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b"
 dependencies = [
  "bumpalo",
  "log",
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
+checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -1953,22 +1962,22 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
+checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.65",
+ "syn 2.0.79",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
+checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
 
 [[package]]
 name = "which"
@@ -2000,11 +2009,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
 [[package]]
 name = "winapi-util"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b"
+checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -2044,27 +2053,36 @@ dependencies = [
  "windows-targets",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets",
+]
+
 [[package]]
 name = "windows-targets"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
 dependencies = [
  "windows_aarch64_gnullvm",
- "windows_aarch64_msvc 0.52.5",
- "windows_i686_gnu 0.52.5",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
  "windows_i686_gnullvm",
- "windows_i686_msvc 0.52.5",
- "windows_x86_64_gnu 0.52.5",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
  "windows_x86_64_gnullvm",
- "windows_x86_64_msvc 0.52.5",
+ "windows_x86_64_msvc 0.52.6",
 ]
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
 
 [[package]]
 name = "windows_aarch64_msvc"
@@ -2074,9 +2092,9 @@ checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
 
 [[package]]
 name = "windows_i686_gnu"
@@ -2086,15 +2104,15 @@ checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
 
 [[package]]
 name = "windows_i686_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
 
 [[package]]
 name = "windows_i686_msvc"
@@ -2104,9 +2122,9 @@ checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
 
 [[package]]
 name = "windows_x86_64_gnu"
@@ -2116,15 +2134,15 @@ checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
 
 [[package]]
 name = "windows_x86_64_msvc"
@@ -2134,9 +2152,9 @@ checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
diff --git a/retis-events/src/events.rs b/retis-events/src/events.rs
index c13cc1130..7c5cff903 100644
--- a/retis-events/src/events.rs
+++ b/retis-events/src/events.rs
@@ -120,6 +120,11 @@ impl Event {
 
         serde_json::Value::Object(event)
     }
+
+    /// Iterator over the existing sections
+    pub fn sections(&self) -> impl Iterator<Item = SectionId> + '_ {
+        self.0.keys().map(|s| s.to_owned())
+    }
 }
 
 impl EventFmt for Event {
diff --git a/retis-events/src/python.rs b/retis-events/src/python.rs
index 561d3816e..49fd4ecaf 100644
--- a/retis-events/src/python.rs
+++ b/retis-events/src/python.rs
@@ -9,7 +9,7 @@ use anyhow::Result;
 use pyo3::{
     exceptions::{PyKeyError, PyRuntimeError},
     prelude::*,
-    types::PyDict,
+    types::{PyDict, PyList},
 };
 
 use super::*;
@@ -29,6 +29,8 @@ use super::*;
 /// is implemented for all event section types is `raw()`, which returns
 /// the data as a dictionary.
 ///
+/// Also, sections can be iterated through the `sections()` helper.
+///
 /// ## Examples
 ///
 /// ```text
@@ -104,6 +106,12 @@ impl PyEvent {
         let format = crate::DisplayFormat::new().multiline(true);
         format!("{}", self.0.display(&format, &crate::FormatterConf::new()))
     }
+
+    /// Returns a list of existing section names.
+    pub fn sections(&self, py: Python<'_>) -> PyResult<Py<PyList>> {
+        let sections: Vec<&str> = self.0.sections().map(|s| s.to_str()).collect();
+        PyList::new_bound(py, sections).extract()
+    }
 }
 
 /// Python event reader

From e478159d492d93a9743c99bc8c52a497a7b36272 Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Wed, 21 Aug 2024 12:59:44 +0200
Subject: [PATCH 11/16] events: support Series in FileEventFactory

Currently, EventSeries are created, printed and even encoded into json
but we do not support reading it back from json.

This patch adds support for EventSeries in FileEventFactory.
The type of file that is being processed is autodetected upon creation
and a differnt interface (next_series()) is used to retrieve EventSeries
so that typing is clear.

By also adding TrackingInfo to the types that unmarshal, we can now do:

```
retis collect [...]
retis sort -o sorted.json
retis print sorted.json
```

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 retis-events/src/events.rs     |  33 +++++-
 retis-events/src/file.rs       |  79 ++++++++++++--
 retis-events/src/python.rs     | 181 ++++++++++++++++++++++++++++++++-
 retis/src/process/cli/print.rs |  35 +++++--
 retis/src/process/cli/sort.rs  |   5 +
 5 files changed, 305 insertions(+), 28 deletions(-)

diff --git a/retis-events/src/events.rs b/retis-events/src/events.rs
index 7c5cff903..8d1a71e0b 100644
--- a/retis-events/src/events.rs
+++ b/retis-events/src/events.rs
@@ -53,13 +53,11 @@ impl Event {
         Event::default()
     }
 
-    pub fn from_json(line: String) -> Result<Event> {
+    /// Create an Event from a json object.
+    pub(crate) fn from_json_obj(mut obj: HashMap<String, serde_json::Value>) -> Result<Event> {
         let mut event = Event::new();
 
-        let mut event_js: HashMap<String, serde_json::Value> = serde_json::from_str(line.as_str())
-            .map_err(|e| anyhow!("Failed to parse json event at line {line}: {e}"))?;
-
-        for (owner, value) in event_js.drain() {
+        for (owner, value) in obj.drain() {
             let parser = event_sections()?
                 .get(&owner)
                 .ok_or_else(|| anyhow!("json contains an unsupported event {}", owner))?;
@@ -73,6 +71,14 @@ impl Event {
         Ok(event)
     }
 
+    /// Create an Event from a json string.
+    pub(crate) fn from_json(line: String) -> Result<Event> {
+        let event_js: HashMap<String, serde_json::Value> = serde_json::from_str(line.as_str())
+            .map_err(|e| anyhow!("Failed to parse json event at line {line}: {e}"))?;
+
+        Self::from_json_obj(event_js)
+    }
+
     /// Insert a new event field into an event.
     pub fn insert_section(
         &mut self,
@@ -301,6 +307,7 @@ fn event_sections() -> Result<&'static EventSectionMap> {
         insert_section!(events, NftEvent);
         insert_section!(events, CtEvent);
         insert_section!(events, StartupEvent);
+        insert_section!(events, TrackingInfo);
 
         Ok(events)
     })
@@ -367,9 +374,25 @@ pub struct EventSeries {
 }
 
 impl EventSeries {
+    /// Encode the EventSeries into a json object.
     pub fn to_json(&self) -> serde_json::Value {
         serde_json::Value::Array(self.events.iter().map(|e| e.to_json()).collect())
     }
+
+    /// Create an EventSeries from a json string.
+    pub(crate) fn from_json(line: String) -> Result<EventSeries> {
+        let mut series = EventSeries::default();
+
+        let mut series_js: Vec<HashMap<String, serde_json::Value>> =
+            serde_json::from_str(line.as_str())
+                .map_err(|e| anyhow!("Failed to parse json series at line {line}: {e}"))?;
+
+        for obj in series_js.drain(..) {
+            let event = Event::from_json_obj(obj)?;
+            series.events.push(event);
+        }
+        Ok(series)
+    }
 }
 
 #[cfg(feature = "test-events")]
diff --git a/retis-events/src/file.rs b/retis-events/src/file.rs
index 387cf3db5..59aa94543 100644
--- a/retis-events/src/file.rs
+++ b/retis-events/src/file.rs
@@ -2,18 +2,28 @@
 
 use std::{
     fs::File,
-    io::{BufRead, BufReader},
+    io::{BufRead, BufReader, Seek},
     path::Path,
 };
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, bail, Result};
 
-use super::Event;
+use super::{Event, EventSeries};
+
+// Type of file that is being processed.
+#[derive(Debug, Clone)]
+pub enum FileType {
+    /// File contains events.
+    Event,
+    /// File contains event series.
+    Series,
+}
 
 /// File events factory retrieving and unmarshaling events
 /// parts.
 pub struct FileEventsFactory {
     reader: BufReader<File>,
+    filetype: FileType,
 }
 
 impl FileEventsFactory {
@@ -21,27 +31,74 @@ impl FileEventsFactory {
     where
         P: AsRef<Path>,
     {
-        Ok(FileEventsFactory {
-            reader: BufReader::new(
-                File::open(&file)
-                    .map_err(|e| anyhow!("Could not open {}: {e}", file.as_ref().display()))?,
-            ),
-        })
+        let mut reader = BufReader::new(
+            File::open(&file)
+                .map_err(|e| anyhow!("Could not open {}: {e}", file.as_ref().display()))?,
+        );
+        let filetype = Self::detect_type(&mut reader)?;
+
+        Ok(FileEventsFactory { reader, filetype })
     }
 }
 
 impl FileEventsFactory {
     /// Retrieve the next event or None if we've reached the end of the file.
+    /// It returns an error if the file contains sorted EventSeries.
     pub fn next_event(&mut self) -> Result<Option<Event>> {
+        match self.filetype {
+            FileType::Event => (),
+            FileType::Series => bail!("Cannot read event from sorted file"),
+        }
+        let mut line = String::new();
+
+        match self.reader.read_line(&mut line) {
+            Err(e) => Err(e.into()),
+            Ok(0) => Ok(None),
+            Ok(_) => Ok(Some(Event::from_json(line)?)),
+        }
+    }
+
+    /// Retrieve the next series or None if we've reached the end of the file.
+    /// It returns an error if the file contains unsorted Events.
+    pub fn next_series(&mut self) -> Result<Option<EventSeries>> {
+        match self.filetype {
+            FileType::Event => bail!("Cannot read series from unsorted file"),
+            FileType::Series => (),
+        }
         let mut line = String::new();
 
         match self.reader.read_line(&mut line) {
+            Err(e) => Err(e.into()),
+            Ok(0) => Ok(None),
+            Ok(_) => Ok(Some(EventSeries::from_json(line)?)),
+        }
+    }
+
+    fn detect_type<T>(reader: &mut T) -> Result<FileType>
+    where
+        T: BufRead + Seek,
+    {
+        let mut line = String::new();
+
+        match reader.read_line(&mut line) {
             Err(e) => return Err(e.into()),
-            Ok(0) => return Ok(None),
+            Ok(0) => return Err(anyhow!("File is empty")),
             Ok(_) => (),
         }
+        reader.rewind()?;
+
+        let first: serde_json::Value = serde_json::from_str(line.as_str())
+            .map_err(|e| anyhow!("Failed to parse event file: {:?}", e))?;
+
+        match first {
+            serde_json::Value::Object(_) => Ok(FileType::Event),
+            serde_json::Value::Array(_) => Ok(FileType::Series),
+            _ => bail!("File contains invalid json data"),
+        }
+    }
 
-        Ok(Some(Event::from_json(line)?))
+    pub fn file_type(&self) -> &FileType {
+        &self.filetype
     }
 }
 
diff --git a/retis-events/src/python.rs b/retis-events/src/python.rs
index 49fd4ecaf..ff7e2d452 100644
--- a/retis-events/src/python.rs
+++ b/retis-events/src/python.rs
@@ -114,15 +114,96 @@ impl PyEvent {
     }
 }
 
+/// Python representation of an EventSeries.
+///
+/// When interacting with sorted event files (the ones generated by
+/// `retis sort`), the file contains lists of events that form an
+/// EventSeries.
+///
+/// # Accessing events
+///
+/// The EventSeries binding implements the iterator protocol so it's easy to
+/// iterate over the internal events.
+///
+/// ## Examples
+///
+/// ```text
+/// >>> len(series)
+/// 3
+/// >>> for event in series:
+///     print(event.show())
+/// ```
+///
+#[pyclass(name = "EventSeries")]
+pub(crate) struct PyEventSeries {
+    events: Vec<Py<PyEvent>>,
+    idx: usize,
+}
+
+impl PyEventSeries {
+    pub(crate) fn new(py: Python<'_>, mut series: EventSeries) -> PyResult<Self> {
+        let mut events = Vec::new();
+        series.events.drain(..).try_for_each(|e| -> PyResult<()> {
+            events.push(Py::new(py, PyEvent::new(e))?);
+            Ok(())
+        })?;
+        Ok(Self { events, idx: 0 })
+    }
+}
+
+#[pymethods]
+impl PyEventSeries {
+    /// Controls how the PyEventSeries is represented, eg. what is the output of
+    /// `print(e)`.
+    fn __repr__(&'_ self, py: Python<'_>) -> PyResult<String> {
+        let n_events = self.events.len();
+        let timestamp = self
+            .events
+            .first()
+            .ok_or_else(|| PyRuntimeError::new_err("Malformed Series with < 1 events"))?
+            .try_borrow(py)?
+            .0
+            .get_section::<CommonEvent>(SectionId::Common)
+            .unwrap()
+            .timestamp;
+        Ok(format!(
+            "PyEventSeries(ts = {}, n_events = {})",
+            timestamp, n_events
+        ))
+    }
+
+    // Implementation of the iterator protocol.
+    fn __iter__(mut slf: PyRefMut<'_, Self>) -> PyRefMut<'_, Self> {
+        slf.idx = 0;
+        slf
+    }
+    pub(crate) fn __next__(&mut self, py: Python<'_>) -> PyResult<Option<Py<PyEvent>>> {
+        if self.idx < self.events.len() {
+            let event = self.events[self.idx].clone_ref(py);
+            self.idx += 1;
+            Ok(Some(event))
+        } else {
+            Ok(None)
+        }
+    }
+
+    /// Returns the number of events in the series.
+    fn __len__(&self) -> usize {
+        self.events.len()
+    }
+}
+
 /// Python event reader
 ///
 /// Objects of this class can read events from unsorted event files.
 ///
+///
 /// ## Example
 ///
 /// ```python
 /// reader = EventReader("retis.data")
-/// for event in reader:
+///
+/// for event in series:
 ///     print(event.show())
 /// ```
 #[pyclass(name = "EventReader")]
@@ -136,6 +217,12 @@ impl PyEventReader {
     pub(crate) fn new(path: PathBuf) -> PyResult<Self> {
         let factory = file::FileEventsFactory::new(path)
             .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+
+        if matches!(factory.file_type(), file::FileType::Series) {
+            return Err(PyRuntimeError::new_err(
+                "Cannot create a EventReader from a sorted file. Use an SeriesReader instead",
+            ));
+        }
         Ok(PyEventReader { factory })
     }
 
@@ -144,6 +231,7 @@ impl PyEventReader {
         slf
     }
 
+    // Return the next Event.
     pub(crate) fn __next__(
         mut slf: PyRefMut<'_, Self>,
         py: Python<'_>,
@@ -162,33 +250,116 @@ impl PyEventReader {
     }
 }
 
+/// Python series reader
+///
+/// Objects of this class can read events from unsorted event files.
+///
+///
+/// ## Example
+///
+/// ```python
+/// reader = EventReader("retis.data")
+///
+/// for event in series:
+///     print(event.show())
+/// ```
+#[pyclass(name = "SeriesReader")]
+pub(crate) struct PySeriesReader {
+    factory: file::FileEventsFactory,
+}
+
+#[pymethods]
+impl PySeriesReader {
+    #[new]
+    pub(crate) fn new(path: PathBuf) -> PyResult<Self> {
+        let factory = file::FileEventsFactory::new(path)
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+
+        if matches!(factory.file_type(), file::FileType::Event) {
+            return Err(PyRuntimeError::new_err(
+                "Cannot create a SeriesReader from an unsorted file. Use an EventReader instead",
+            ));
+        }
+        Ok(PySeriesReader { factory })
+    }
+
+    // Implementation of the iterator protocol.
+    pub(crate) fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
+        slf
+    }
+
+    // Return the next EventSeries.
+    pub(crate) fn __next__(
+        mut slf: PyRefMut<'_, Self>,
+        py: Python<'_>,
+    ) -> PyResult<Option<Py<PyAny>>> {
+        match slf
+            .factory
+            .next_series()
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?
+        {
+            Some(series) => {
+                let pyseries: Bound<'_, PyEventSeries> =
+                    Bound::new(py, PyEventSeries::new(py, series)?)?;
+                Ok(Some(pyseries.into_any().into()))
+            }
+            None => Ok(None),
+        }
+    }
+}
+
 /// Python event file
 ///
 /// Objects of this class can read files generated by retis and create
-/// EventReader instances to iterate over their content.
+/// EventReader and SeriesReader instances to iterate over their content.
 ///
 /// ## Example
 ///
 /// ```python
 /// event_file = EventFile("retis.data")
-/// for event in event_file.events():
-///     print(event.show())
+/// if event_file.sorted():
+///     for series in event_file.series():
+///         for event in series:
+///             print(event.show())
+///
+/// else:
+///     for event in event_file.events():
+///         print(event.show())
 /// ```
 #[pyclass(name = "EventFile")]
 pub(crate) struct PyEventFile {
     path: PathBuf,
+    ftype: file::FileType,
 }
 
 #[pymethods]
 impl PyEventFile {
     #[new]
     pub(crate) fn new(path: PathBuf) -> PyResult<Self> {
-        Ok(PyEventFile { path })
+        let temp = file::FileEventsFactory::new(&path)
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+        let ftype = temp.file_type();
+        Ok(PyEventFile {
+            path,
+            ftype: ftype.clone(),
+        })
+    }
+
+    // Returns whether the file is sorted.
+    pub(crate) fn sorted(&self) -> PyResult<bool> {
+        match self.ftype {
+            file::FileType::Series => Ok(true),
+            file::FileType::Event => Ok(false),
+        }
     }
 
     pub(crate) fn events(&self) -> PyResult<PyEventReader> {
         PyEventReader::new(self.path.clone())
     }
+
+    pub(crate) fn series(&self) -> PyResult<PySeriesReader> {
+        PySeriesReader::new(self.path.clone())
+    }
 }
 
 /// Converts a serde_json::Value to a PyObject.
diff --git a/retis/src/process/cli/print.rs b/retis/src/process/cli/print.rs
index 0550864fb..813ca9a56 100644
--- a/retis/src/process/cli/print.rs
+++ b/retis/src/process/cli/print.rs
@@ -10,7 +10,10 @@ use clap::Parser;
 
 use crate::{
     cli::*,
-    events::{file::FileEventsFactory, *},
+    events::{
+        file::{FileEventsFactory, FileType},
+        *,
+    },
     helpers::signals::Running,
     module::Modules,
     process::display::*,
@@ -48,15 +51,33 @@ impl SubCommandParserRunner for Print {
                 TimeFormat::MonotonicTimestamp
             });
 
-        // Formatter & printer for events.
-        let mut output = PrintEvent::new(Box::new(stdout()), PrintEventFormat::Text(format));
+        match factory.file_type() {
+            FileType::Event => {
+                // Formatter & printer for events.
+                let mut event_output =
+                    PrintEvent::new(Box::new(stdout()), PrintEventFormat::Text(format));
 
-        while run.running() {
-            match factory.next_event()? {
-                Some(event) => output.process_one(&event)?,
-                None => break,
+                while run.running() {
+                    match factory.next_event()? {
+                        Some(event) => event_output.process_one(&event)?,
+                        None => break,
+                    }
+                }
+            }
+            FileType::Series => {
+                // Formatter & printer for series.
+                let mut series_output =
+                    PrintSeries::new(Box::new(stdout()), PrintEventFormat::Text(format));
+
+                while run.running() {
+                    match factory.next_series()? {
+                        Some(series) => series_output.process_one(&series)?,
+                        None => break,
+                    }
+                }
             }
         }
+
         Ok(())
     }
 }
diff --git a/retis/src/process/cli/sort.rs b/retis/src/process/cli/sort.rs
index 6adc9c3b0..6cc65b2bc 100644
--- a/retis/src/process/cli/sort.rs
+++ b/retis/src/process/cli/sort.rs
@@ -71,6 +71,11 @@ impl SubCommandParserRunner for Sort {
         // Create event factory.
         let mut factory = FileEventsFactory::new(self.input.as_path())?;
 
+        if matches!(factory.file_type(), file::FileType::Series) {
+            println!("File already sorted");
+            return Ok(());
+        }
+
         let mut series = EventSorter::new();
         let mut tracker = AddTracking::new();
         let mut printers = Vec::new();

From 33985cf2409ca85df521c99226df47684d029a62 Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Tue, 17 Sep 2024 16:04:30 +0200
Subject: [PATCH 12/16] retis-events: introduce python library.

The features used for pyo3 are different (and not-compatible)
for the embedded interpreter and python bindings, so move the
interpreter into an independent file that is easy to conditionally
compile off.

Use maturin to build with stable abi3.

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 Makefile                          |  8 +++++-
 retis-events/Cargo.toml           |  6 ++--
 retis-events/PYTHON_LIB_README.md | 16 +++++++++++
 retis-events/pyproject.toml       | 35 ++++++++++++++++++++++
 retis-events/src/lib.rs           | 16 +++++++++++
 retis-events/src/python.rs        | 48 ++-----------------------------
 retis-events/src/python_embed.rs  | 47 ++++++++++++++++++++++++++++++
 retis/Cargo.toml                  |  2 +-
 retis/src/process/cli/python.rs   |  2 +-
 9 files changed, 130 insertions(+), 50 deletions(-)
 create mode 100644 retis-events/PYTHON_LIB_README.md
 create mode 100644 retis-events/pyproject.toml
 create mode 100644 retis-events/src/python_embed.rs

diff --git a/Makefile b/Makefile
index a07dc3d2a..a2ce45656 100644
--- a/Makefile
+++ b/Makefile
@@ -13,6 +13,7 @@ RELEASE_NAME ?= $(shell $(CARGO) metadata --no-deps --format-version=1 | jq -r '
 export ARCH CFLAGS CLANG LCC OBJCOPY RELEASE_NAME RELEASE_VERSION RUSTFLAGS
 
 PRINT = printf
+CONTAINER_RUNTIME := podman
 
 VERBOSITY := $(filter 1,$(V))
 
@@ -97,6 +98,10 @@ $(EBPF_PROBES) $(EBPF_HOOKS): $(LIBBPF_INCLUDES)
 	CFLAGS_INCLUDES="$(INCLUDES)" \
 	$(MAKE) -r -f $(ROOT_DIR)/ebpf.mk -C $@
 
+pylib:
+	$(call out_console,MATURIN,Building python bindings ...)
+	$(CONTAINER_RUNTIME) run --rm --name retis_build_maturin -v $$PWD:/io:z ghcr.io/pyo3/maturin build -m retis-events/Cargo.toml -F python-lib
+
 clean-ebpf:
 	$(call out_console,CLEAN,cleaning ebpf progs...)
 	for i in $(EBPF_PROBES) $(EBPF_HOOKS); do \
@@ -121,6 +126,7 @@ help:
 	$(PRINT) 'install             --  Installs Retis.'
 	$(PRINT) 'release             --  Builds Retis with the release option.'
 	$(PRINT) 'test                --  Builds and runs unit tests.'
+	$(PRINT) 'pylib 	      --  Builds the python bindings.'
 	$(PRINT)
 	$(PRINT) 'Optional variables that can be used to override the default behavior:'
 	$(PRINT) 'V                   --  If set to 1 the verbose output will be printed.'
@@ -136,4 +142,4 @@ help:
 	$(PRINT) 'NOVENDOR            --  Avoid to self detect and consume the vendored headers'
 	$(PRINT) '                        shipped with libbpf-sys.'
 
-.PHONY: all bench clean clean-ebpf ebpf $(EBPF_PROBES) $(EBPF_HOOKS) help install release test
+.PHONY: all bench clean clean-ebpf ebpf $(EBPF_PROBES) $(EBPF_HOOKS) help install release test pylib
diff --git a/retis-events/Cargo.toml b/retis-events/Cargo.toml
index f6a96ed5a..10afaede2 100644
--- a/retis-events/Cargo.toml
+++ b/retis-events/Cargo.toml
@@ -5,10 +5,12 @@ edition = "2021"
 
 [features]
 python = ["dep:pyo3", "retis-derive/python"]
+python-embed = ["python", "pyo3/auto-initialize"]
+python-lib = ["python", "pyo3/extension-module", "pyo3/abi3-py38"]
 test-events = []
 
 [lib]
-crate-type = ["lib"]
+crate-type = ["lib", "cdylib"]
 
 [dependencies]
 anyhow = "1.0"
@@ -17,7 +19,7 @@ chrono = "0.4"
 log = { version = "0.4", features = ["std"] }
 once_cell = "1.15"
 retis-derive = {version = "1.4", path = "../retis-derive"}
-pyo3 = {version = "0.22", features = ["gil-refs", "auto-initialize", "multiple-pymethods"], optional = true}
+pyo3 = {version = "0.22", features = ["gil-refs", "multiple-pymethods"], optional = true}
 serde = {version = "1.0", features = ["derive"]}
 serde_json = "1.0"
 serde_with = "3.0"
diff --git a/retis-events/PYTHON_LIB_README.md b/retis-events/PYTHON_LIB_README.md
new file mode 100644
index 000000000..46d808d39
--- /dev/null
+++ b/retis-events/PYTHON_LIB_README.md
@@ -0,0 +1,16 @@
+# retis
+
+Python bindings for [retis](https://retis.readthedocs.io/en/stable/) events.
+
+This python library can be used to read and post-process retis events.
+
+Example:
+
+```python
+from retis import EventReader, Event
+
+reader = EventReader("retis.data")
+
+for event in reader:
+    print(e.show())
+```
diff --git a/retis-events/pyproject.toml b/retis-events/pyproject.toml
new file mode 100644
index 000000000..0b1fd20f5
--- /dev/null
+++ b/retis-events/pyproject.toml
@@ -0,0 +1,35 @@
+[build-system]
+requires = ["maturin>=1.0"]
+build-backend = "maturin"
+
+[tool.maturin]
+bindings = "pyo3"
+features = ["python-lib"]
+
+[project]
+name = "retis"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+
+    "Operating System :: POSIX :: Linux",
+
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python",
+    "Programming Language :: Rust",
+
+    "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
+]
+description = "Python bindings for Retis events"
+readme = "PYTHON_LIB_README.md"
+
+[project.urls]
+HomePage = "https://github.com/retis-org/retis"
+Documentation = "https://retis.readthedocs.io/"
+Repository = "https://github.com/retis-org/retis"
diff --git a/retis-events/src/lib.rs b/retis-events/src/lib.rs
index 6bcdda709..bff3b7d02 100644
--- a/retis-events/src/lib.rs
+++ b/retis-events/src/lib.rs
@@ -16,6 +16,8 @@ pub mod file;
 pub mod net;
 #[cfg(feature = "python")]
 pub mod python;
+#[cfg(feature = "python-embed")]
+pub mod python_embed;
 
 pub mod common;
 pub use common::*;
@@ -40,3 +42,17 @@ pub use user::*;
 
 // Re-export derive macros.
 use retis_derive::*;
+
+#[cfg(feature = "python-lib")]
+use pyo3::prelude::*;
+
+#[cfg(feature = "python-lib")]
+#[pymodule]
+fn retis(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+    m.add_class::<python::PyEvent>()?;
+    m.add_class::<python::PyEventSeries>()?;
+    m.add_class::<python::PyEventReader>()?;
+    m.add_class::<python::PySeriesReader>()?;
+    m.add_class::<python::PyEventFile>()?;
+    Ok(())
+}
diff --git a/retis-events/src/python.rs b/retis-events/src/python.rs
index ff7e2d452..d76e6a468 100644
--- a/retis-events/src/python.rs
+++ b/retis-events/src/python.rs
@@ -3,13 +3,12 @@
 //! This module contains python bindings for retis events so that they can
 //! be inspected in post-processing tools written in python.
 
-use std::{collections::HashMap, fs, path::PathBuf, str::FromStr};
+use std::{collections::HashMap, path::PathBuf, str::FromStr};
 
-use anyhow::Result;
 use pyo3::{
     exceptions::{PyKeyError, PyRuntimeError},
     prelude::*,
-    types::{PyDict, PyList},
+    types::PyList,
 };
 
 use super::*;
@@ -208,7 +207,7 @@ impl PyEventSeries {
 /// ```
 #[pyclass(name = "EventReader")]
 pub(crate) struct PyEventReader {
-    factory: file::FileEventsFactory,
+    pub(crate) factory: file::FileEventsFactory,
 }
 
 #[pymethods]
@@ -385,44 +384,3 @@ pub(crate) fn to_pyobject(val: &serde_json::Value, py: Python<'_>) -> PyObject {
         }
     }
 }
-
-/// Create a python shell and execute the provided script.
-pub fn shell_execute(file: PathBuf, script: Option<&PathBuf>) -> Result<()> {
-    let event_file = PyEventFile::new(file)?;
-
-    Python::with_gil(|py| -> PyResult<()> {
-        let shell = PyShell::new(py, event_file)?;
-        if let Some(script) = script {
-            shell.run(&fs::read_to_string(script)?)
-        } else {
-            shell.interact()
-        }
-    })?;
-    Ok(())
-}
-
-/// Python shell.
-struct PyShell<'a> {
-    py: Python<'a>,
-    globals: Bound<'a, PyDict>,
-}
-
-impl<'a> PyShell<'a> {
-    const INTERACTIVE_SHELL: &'static str = "import code; code.interact(local=locals())";
-
-    fn new(py: Python<'a>, file: PyEventFile) -> PyResult<Self> {
-        let globals = PyDict::new_bound(py);
-        globals.set_item("reader", Py::new(py, file)?.into_bound(py))?;
-
-        Ok(Self { py, globals })
-    }
-
-    fn run(&self, script: &str) -> PyResult<()> {
-        self.py
-            .run_bound(script, Some(&self.globals.as_borrowed()), None)
-    }
-
-    fn interact(&self) -> PyResult<()> {
-        self.run(Self::INTERACTIVE_SHELL)
-    }
-}
diff --git a/retis-events/src/python_embed.rs b/retis-events/src/python_embed.rs
new file mode 100644
index 000000000..8e1add508
--- /dev/null
+++ b/retis-events/src/python_embed.rs
@@ -0,0 +1,47 @@
+use std::{fs, path::PathBuf};
+
+use anyhow::Result;
+use pyo3::{prelude::*, types::PyDict};
+
+use super::python::*;
+
+/// Create a python shell and execute the provided script.
+pub fn shell_execute(file: PathBuf, script: Option<&PathBuf>) -> Result<()> {
+    let event_file = PyEventFile::new(file)?;
+
+    Python::with_gil(|py| -> PyResult<()> {
+        let shell = PyShell::new(py, event_file)?;
+        if let Some(script) = script {
+            shell.run(&fs::read_to_string(script)?)
+        } else {
+            shell.interact()
+        }
+    })?;
+    Ok(())
+}
+
+/// Python shell.
+struct PyShell<'a> {
+    py: Python<'a>,
+    globals: Bound<'a, PyDict>,
+}
+
+impl<'a> PyShell<'a> {
+    const INTERACTIVE_SHELL: &'static str = "import code; code.interact(local=locals())";
+
+    fn new(py: Python<'a>, file: PyEventFile) -> PyResult<Self> {
+        let globals = PyDict::new_bound(py);
+        globals.set_item("reader", Py::new(py, file)?.into_bound(py))?;
+
+        Ok(Self { py, globals })
+    }
+
+    fn run(&self, script: &str) -> PyResult<()> {
+        self.py
+            .run_bound(script, Some(&self.globals.as_borrowed()), None)
+    }
+
+    fn interact(&self) -> PyResult<()> {
+        self.run(Self::INTERACTIVE_SHELL)
+    }
+}
diff --git a/retis/Cargo.toml b/retis/Cargo.toml
index ab54c19fd..7d34d1070 100644
--- a/retis/Cargo.toml
+++ b/retis/Cargo.toml
@@ -33,7 +33,7 @@ caps = "0.5"
 clap = { version = "4.0", features = ["derive", "string"] }
 clap_complete = "4.4"
 elf = "0.7"
-events = {version = "1.4", path = "../retis-events", package="retis-events", features=["python"]}
+events = {version = "1.4", path = "../retis-events", package="retis-events", features=["python-embed"]}
 flate2 = "1.0"
 libbpf-rs = "0.22"
 libbpf-sys = "=1.2"
diff --git a/retis/src/process/cli/python.rs b/retis/src/process/cli/python.rs
index 68924e87f..5250543c1 100644
--- a/retis/src/process/cli/python.rs
+++ b/retis/src/process/cli/python.rs
@@ -3,7 +3,7 @@ use std::path::PathBuf;
 use anyhow::Result;
 use clap::{arg, Parser};
 
-use crate::{cli::*, events::python::shell_execute, module::Modules};
+use crate::{cli::*, events::python_embed::shell_execute, module::Modules};
 
 /// Runs Python scripts with events imported.
 #[derive(Parser, Debug, Default)]

From 484f634968c0aab44ae6aba54b56782ea42e140e Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Thu, 19 Sep 2024 11:17:48 +0200
Subject: [PATCH 13/16] ci: add a cirrus task to build python bindings

Store the result as artifact to ease testing.

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 .cirrus.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.cirrus.yml b/.cirrus.yml
index df62c6891..5ccf8745f 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -56,6 +56,14 @@ python_task:
     - python3 -m pip install black
     - python3 -m black --check --diff tests/*.py
 
+build_python_lib_task:
+  container:
+    image: ghcr.io/pyo3/maturin
+  build_script:
+    - maturin build --release -m retis-events/Cargo.toml -F python-lib  
+  dist_artifacts:
+    path: "target/wheels/*"
+
 functional_task:
   # Run for PRs with a specific label
   required_pr_labels: run-functional-tests

From ef9946f116839780358f95c00518f72bae33b1e1 Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Wed, 18 Sep 2024 20:42:27 +0200
Subject: [PATCH 14/16] docs: Document python bindings

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 RELEASE.md           |   6 ++-
 docs/CONTRIBUTING.md |  14 ++++++
 docs/index.md        |   4 ++
 docs/python.md       | 104 +++++++++++++++++++++++++++++++++++++++++++
 mkdocs.yml           |   1 +
 5 files changed, 127 insertions(+), 2 deletions(-)
 create mode 100644 docs/python.md

diff --git a/RELEASE.md b/RELEASE.md
index 7de22e100..35bbd383a 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -5,8 +5,8 @@
    1. Make sure the README is up-to-date (collectors, build instructions, etc).
    1. Add new authors to the authors file if needed, by running
       `./tools/authors.sh` and committing the changes.
-   1. Update the version in `Cargo.toml`.
-   1. Update the version name in `src/main.rs`.
+   1. Update the version in `Cargo.toml`, retis-events/Cargo.toml and
+      retis-derive/Cargo.toml.
    1. Run `cargo publish --dry-run` to check for any issue.
    1. Open a PR and get it merged. This must have runtime tests enabled!
    1. Tag the right commit in the `vx.y.z` form and push it.
@@ -21,6 +21,8 @@
       1. Manually run the workflow [Build and push container image](https://github.com/retis-org/retis/actions/workflows/build_push_image.yaml)
          (in the Actions tab on the Retis Github page) selecting the branch and setting the
          `release_tags` with the space separated list of tags (i.e. release_tags="x.y.z latest").
+   1. Build and upload the python bindings.
+      1. `podman run --rm --env MATURIN_PYPI_TOKEN=$(cat ~/.my_pypi_token) -v $(pwd):/io:z ghcr.io/pyo3/maturin publish -m retis-events/Cargo.toml -F python-lib`
 1. Release on [crates.io](https://crates.io): `cargo publish`.
 1. Write and publish a release notes in the GitHub interface. This must be done
    once the rpm and the image successfully built to allow pushing last minute
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 94d6b9764..d2b76a41d 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -122,6 +122,20 @@ flavor coding style for the BPF parts.
    latest `main`. This can happen at any time, e.g. when other pull-requests are
    being merged.
 
+### Python bindings development
+
+When changing python bindings, it can be useful to quickly build and install the
+current code into a virtual environment for testing. An easy way to do it is by
+using [maturin](https://github.com/PyO3/maturin):
+
+```
+$ python -m venv venv && source venv/bin/activate # jump into a virtual env (required)
+$ pip install maturin
+$ maturin develop -m retis-events/Cargo.toml -F python-lib
+$ python
+>>> import retis
+```
+
 ### Documentation preview
 
 HTTP documentation is automatically generated for releases and the
diff --git a/docs/index.md b/docs/index.md
index d21b0b027..dbd7a93a7 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -121,6 +121,10 @@ $ PAGER=more retis sort
 $ NOPAGER=1 retis sort
 ```
 
+In addition to built-in post-processing commands, it is possible to use the
+python bindings to implement custom processing. See the
+[python bindings documentation](python.md).
+
 ## Profiles and customization
 
 Retis has the concept of profiles, which are a predefined set of cli arguments
diff --git a/docs/python.md b/docs/python.md
new file mode 100644
index 000000000..945c6f06f
--- /dev/null
+++ b/docs/python.md
@@ -0,0 +1,104 @@
+# Python bindings
+
+Besides the basic post-processing commands provided by `retis` (e.g: `sort`),
+python bindings exist to enable writing custom post-processing scripts.
+
+These bindings can be used in two different ways: the built-in python
+interpreter and the external python library.
+
+## Overview
+
+Python bindings currently provide the following basic python classes that
+allow inspecting retis events:
+
+- **Event**: Python representation of a retis event. It provides helpers to
+access the event's sections and data within those sections.
+- **EventSeries**: Python representation of a series of sorted events resulting
+from the execution of `retis sort -o`. It implements the iterator protocol to
+access the events.
+- **EventReader**: Class capable of reading a file created by `retis collect`
+and iterate over its events.
+- **SeriesReader**: Class capable of reading a file created by `retis sort`
+and iterate over the its series.
+- **EventFile**: Reads an event file, determines whether it is sorted or not
+and allow the creation of `EventReader` and `SeriesReader` instances.
+
+More details can be found in the `retis_events` crate documentation.
+
+## Builtin python interpreter
+
+The builtin interpreter enables the execution of python scripts that can inspect
+and further process retis events.
+
+Within the script execution context, a global variable called `reader` is
+available. It is of type `EventFile`.
+
+
+```python
+$ cat myscript.py
+for event in reader.events():
+    if "skb" in event and getattr(event["skb"], "tcp", None):
+        print("TCP event with dport: {}".format(
+            event["skb"].tcp.dport))
+
+$ retis python myscript.py
+```
+
+If no script is provided, an interactive shell is created. Example:
+
+```text
+$ retis collect [...]
+$ retis python
+Python 3.12.6 (main, Sep  9 2024, 00:00:00) [GCC 14.2.1 20240801 (Red Hat 14.2.1-1)] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+(InteractiveConsole)
+>>> reader
+<builtins.EventFile object at 0x7f1943606030>
+>>> reader.sorted()
+False
+>>> events = reader.events()
+>>> events 
+<builtins.EventReader object at 0x7f44e17eaf60>
+>>> print("Got {} events".format(sum(1 for _ in events)))
+Got 783 events
+```
+
+The `EventFile` object available can also iterate through sorted files:
+
+```text
+$ retis collect [...]
+$ retis sort -o sorted.data
+$ retis python -i sorted.data
+Python 3.12.6 (main, Sep  9 2024, 00:00:00) [GCC 14.2.1 20240801 (Red Hat 14.2.1-1)] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+(InteractiveConsole)
+>>> reader
+<builtins.EventFile object at 0x7ffb50906130>
+>>> reader.sorted()
+True
+>>> series = reader.series()
+>>> series 
+<builtins.SeriesReader object at 0x7f44e17e5b60>
+>>> print("Got {} series".format(sum(1 for _ in series))
+Got 149 series
+```
+
+## Python library
+
+For more sophisticated programs that require more control over the python
+environment (interpreter, dependencies, etc), a python library is available in
+[pypi](https://pypi.org/retis). Unlike the builtin command, in this case the
+`EventReader` or `SeriesReader` has to be be created manually.
+
+```python
+from retis import SeriesReader
+
+import statistics
+
+reader = SeriesReader("sorted_events.json")
+
+events_per_series = [len(s) for s in reader]
+
+print("Number of series: {}".format(len(events_per_series)))
+print("Average events per series: {}".format(statistics.mean(events_per_series)))
+```
diff --git a/mkdocs.yml b/mkdocs.yml
index f37740e48..35e93bbd3 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -15,6 +15,7 @@ nav:
   - Contributing: CONTRIBUTING.md
   - External resources: resources.md
   - Event sections: event_sections.md
+  - Python bindings: python.md
   - Collectors:
       - skb: modules/skb.md
       - skb-drop: modules/skb_drop.md

From 574ac763d83019507462ef245cbb254c2d1d80b0 Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Thu, 19 Sep 2024 18:08:35 +0200
Subject: [PATCH 15/16] events: add python library tests

Use tox to handle virtual env and pytest to as test runner.
Add a task in cirrus CI to run them.

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 .cirrus.yml                                   | 20 +++++-
 .gitignore                                    |  1 +
 Makefile                                      | 10 ++-
 docs/CONTRIBUTING.md                          |  1 +
 retis-events/pytests/test_reader.py           | 61 +++++++++++++++++++
 .../test_data/test_events_sorted.json         |  3 +
 retis-events/tox.ini                          | 12 ++++
 7 files changed, 106 insertions(+), 2 deletions(-)
 create mode 100644 retis-events/pytests/test_reader.py
 create mode 100644 retis-events/test_data/test_events_sorted.json
 create mode 100644 retis-events/tox.ini

diff --git a/.cirrus.yml b/.cirrus.yml
index 5ccf8745f..dd2caf8cd 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -60,10 +60,28 @@ build_python_lib_task:
   container:
     image: ghcr.io/pyo3/maturin
   build_script:
-    - maturin build --release -m retis-events/Cargo.toml -F python-lib  
+    - maturin build --release -m retis-events/Cargo.toml -F python-lib
   dist_artifacts:
     path: "target/wheels/*"
 
+test_python_lib_task:
+  container:
+    matrix:
+      - image: python:3.8
+      - image: python:3.9
+      - image: python:3.10
+      - image: python:3.11
+      - image: python:3.12
+  setup_script:
+    - apt-get update
+    - apt-get install -y curl
+    - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup-init.sh
+    - chmod +x rustup-init.sh && ./rustup-init.sh -y --profile minimal
+  build_script:
+    - source $HOME/.cargo/env
+    - python -m pip install tox
+    - make pytest
+
 functional_task:
   # Run for PRs with a specific label
   required_pr_labels: run-functional-tests
diff --git a/.gitignore b/.gitignore
index 892586147..8a0abb774 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
 .out
 __pycache__/
 retis.data
+retis-events/.tox
diff --git a/Makefile b/Makefile
index a2ce45656..25cb11529 100644
--- a/Makefile
+++ b/Makefile
@@ -102,6 +102,13 @@ pylib:
 	$(call out_console,MATURIN,Building python bindings ...)
 	$(CONTAINER_RUNTIME) run --rm --name retis_build_maturin -v $$PWD:/io:z ghcr.io/pyo3/maturin build -m retis-events/Cargo.toml -F python-lib
 
+pytest-deps:
+	@which tox &> /dev/null || (echo "Please install tox ('pip install tox')."; exit 1)
+
+pytest: pytest-deps
+	$(call out_console,TOX,Testing python bindings ...)
+	cd retis-events && tox
+
 clean-ebpf:
 	$(call out_console,CLEAN,cleaning ebpf progs...)
 	for i in $(EBPF_PROBES) $(EBPF_HOOKS); do \
@@ -127,6 +134,7 @@ help:
 	$(PRINT) 'release             --  Builds Retis with the release option.'
 	$(PRINT) 'test                --  Builds and runs unit tests.'
 	$(PRINT) 'pylib 	      --  Builds the python bindings.'
+	$(PRINT) 'pytest 	      --  Tests the python bindings (requires "tox" installed).'
 	$(PRINT)
 	$(PRINT) 'Optional variables that can be used to override the default behavior:'
 	$(PRINT) 'V                   --  If set to 1 the verbose output will be printed.'
@@ -142,4 +150,4 @@ help:
 	$(PRINT) 'NOVENDOR            --  Avoid to self detect and consume the vendored headers'
 	$(PRINT) '                        shipped with libbpf-sys.'
 
-.PHONY: all bench clean clean-ebpf ebpf $(EBPF_PROBES) $(EBPF_HOOKS) help install release test pylib
+.PHONY: all bench clean clean-ebpf ebpf $(EBPF_PROBES) $(EBPF_HOOKS) help install release test pylib pytest-deps pytest
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index d2b76a41d..c481f7975 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -100,6 +100,7 @@ flavor coding style for the BPF parts.
    1. `cargo clippy -- -D warnings`
    1. `make test V=1`, or to include runtime tests,
       `CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER=sudo CARGO_CMD_OPTS="--features=test_cap_bpf" make test V=1`
+   1. `make pytest V=1`
 1. Make sure commits are
    [signed off](https://www.kernel.org/doc/html/latest/process/submitting-patches.html?highlight=signed%20off#developer-s-certificate-of-origin-1-1).
 1. Use a clear, concise and descriptive title.
diff --git a/retis-events/pytests/test_reader.py b/retis-events/pytests/test_reader.py
new file mode 100644
index 000000000..b17f78c9b
--- /dev/null
+++ b/retis-events/pytests/test_reader.py
@@ -0,0 +1,61 @@
+from retis import Event, EventFile, EventReader, EventSeries, SeriesReader
+
+import pytest
+
+
+def verify_event(e):
+    """Verify the event is valid"""
+    assert e.__class__ == Event
+    assert isinstance(e.raw(), dict)
+    assert isinstance(e.show(), str)
+    assert "userspace" in e or "kernel" in e
+
+
+def verify_event_reader(r):
+    assert r.__class__ == EventReader
+    for e in r:
+        verify_event(e)
+
+
+def verify_series_reader(r):
+    assert r.__class__ == SeriesReader
+    for s in r:
+        assert s.__class__ == EventSeries
+        length = len(s)
+        i = 0
+
+        for e in s:
+            verify_event(e)
+            i += 1
+
+        assert i == length
+
+
+def test_event_reader():
+    """Test event reader is capable of reading valid events"""
+    r = EventReader("test_data/test_events.json")
+    verify_event_reader(r)
+
+
+def test_series_reader():
+    """Test SeriesReader is capable of reading sorted events"""
+    r = SeriesReader("test_data/test_events_sorted.json")
+    verify_series_reader(r)
+
+
+def test_event_File():
+    """Test EventFile is capable of reading reader is capable generating
+    iterators"""
+    f = EventFile("test_data/test_events.json")
+    assert not f.sorted()
+    verify_event_reader(f.events())
+
+    with pytest.raises(Exception):
+        f.series()
+
+    sf = EventFile("test_data/test_events_sorted.json")
+    assert sf.sorted()
+    verify_series_reader(sf.series())
+
+    with pytest.raises(Exception):
+        sf.events()
diff --git a/retis-events/test_data/test_events_sorted.json b/retis-events/test_data/test_events_sorted.json
new file mode 100644
index 000000000..d6d418fef
--- /dev/null
+++ b/retis-events/test_data/test_events_sorted.json
@@ -0,0 +1,3 @@
+[{"common":{"smp_id":0,"task":{"comm":"napi/phy0-8197","pid":1360,"tgid":1360},"timestamp":23868643385999},"kernel":{"probe_type":"kprobe","symbol":"tcp_v4_rcv"},"skb":{},"skb-tracking":{"orig_head":18446616568293939200,"skb":18446616546229617920,"timestamp":23868643385999},"tracking":{"idx":0,"skb":{"orig_head":18446616568293939200,"skb":18446616546229617920,"timestamp":23868643385999}}}]
+[{"common":{"smp_id":2,"task":{"comm":"napi/phy0-8197","pid":1360,"tgid":1360},"timestamp":23868955262984},"kernel":{"probe_type":"raw_tracepoint","symbol":"openvswitch:ovs_dp_upcall"},"ovs":{"cmd":1,"cpu":1,"event_type":"upcall","port":3366920467},"skb":{},"skb-tracking":{"orig_head":18446616576100907520,"skb":18446616546107689472,"timestamp":23868955262984},"tracking":{"idx":0,"skb":{"orig_head":18446616576100907520,"skb":18446616546107689472,"timestamp":23868955262984}}},{"common":{"smp_id":3,"task":{"comm":"napi/phy0-8197","pid":1360,"tgid":1360},"timestamp":23868955276361},"kernel":{"probe_type":"kretprobe","symbol":"ovs_dp_upcall"},"ovs":{"event_type":"upcall_return","ret":0,"upcall_cpu":1,"upcall_ts":23868955262984},"skb":{},"skb-tracking":{"orig_head":18446616576100907520,"skb":18446616546107689472,"timestamp":23868955262984},"tracking":{"idx":1,"skb":{"orig_head":18446616576100907520,"skb":18446616546107689472,"timestamp":23868955262984}}}]
+[{"common":{"smp_id":0,"task":{"comm":"napi/phy0-8197","pid":1360,"tgid":1360},"timestamp":23868955449721},"kernel":{"probe_type":"raw_tracepoint","symbol":"skb:kfree_skb"},"skb":{},"skb-drop":{"drop_reason":"NO_SOCKET"},"skb-tracking":{"orig_head":18446616575285769216,"skb":18446616552502694400,"timestamp":23868955437572},"tracking":{"idx":0,"skb":{"orig_head":18446616575285769216,"skb":18446616552502694400,"timestamp":23868955437572}}}]
diff --git a/retis-events/tox.ini b/retis-events/tox.ini
new file mode 100644
index 000000000..240d9ce91
--- /dev/null
+++ b/retis-events/tox.ini
@@ -0,0 +1,12 @@
+[tox]
+requires =
+    tox>=4
+env_list = py{38,39,310,311,312}
+
+[testenv]
+description = Run unit tests
+deps =
+    pytest>=7
+    maturin>=1.7
+commands =
+    pytest {posargs:pytests}

From b4f20e04042db081568155b10a0ebd33beb7f044 Mon Sep 17 00:00:00 2001
From: Adrian Moreno <amorenoz@redhat.com>
Date: Mon, 21 Oct 2024 14:38:46 +0200
Subject: [PATCH 16/16] Cargo.toml: make python optional but default.

Python support provided by pyo3 requires linking against python >= 3.6.
Building retis on distros with older python versions would fail. In
order to work around this issues, make python an optional dependency but
enable it by default.

On non-supported distros, CARGO_CMD_OPTS="--no-default-features" can be
used to build without python support.

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 .cirrus.yml                  |   3 +-
 Cargo.lock                   | 183 +++++++++++++++++------------------
 docs/python.md               |   9 ++
 retis/Cargo.toml             |   6 +-
 retis/src/cli/cli.rs         |   1 +
 retis/src/process/cli/mod.rs |   2 +
 6 files changed, 108 insertions(+), 96 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index dd2caf8cd..1325c50ab 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -100,6 +100,7 @@ functional_task:
       env:
         DISTRO: x86_64-c8s
         TEST_EXTRA_ARGS: "--ignore=test_ovs.py --ignore=test_nft.py"
+        CARGO_CMD_OPTS: "--no-default-features"
     - name: Centos 9 Stream (x86_64)
       env:
         DISTRO: x86_64-c9s
@@ -151,7 +152,7 @@ functional_task:
     - ssh ${DISTRO} 'sh -exc "uname -a"'
 
   test_script: &functional_test
-    - ssh -tt ${DISTRO} 'bash --login -exc "cd /vagrant && make V=1"'
+    - ssh -tt ${DISTRO} "env CARGO_CMD_OPTS=$CARGO_CMD_OPTS bash --login -exc 'cd /vagrant && make V=1'"
     - ssh -tt ${DISTRO} "cd /vagrant/tests && sudo python3 -m pytest ${TEST_EXTRA_ARGS}"
 
 # Manual trigger for non-PR branches.
diff --git a/Cargo.lock b/Cargo.lock
index d1f7fbbb1..449baee63 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -83,9 +83,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.89"
+version = "1.0.90"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6"
+checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95"
 
 [[package]]
 name = "ascii"
@@ -113,9 +113,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
 [[package]]
 name = "bindgen"
-version = "0.69.4"
+version = "0.69.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
+checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
 dependencies = [
  "bitflags 2.6.0",
  "cexpr",
@@ -130,7 +130,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.79",
+ "syn 2.0.82",
  "which",
 ]
 
@@ -221,9 +221,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.1.24"
+version = "1.1.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938"
+checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f"
 dependencies = [
  "shlex",
 ]
@@ -277,9 +277,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.19"
+version = "4.5.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615"
+checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -287,9 +287,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.19"
+version = "4.5.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b"
+checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54"
 dependencies = [
  "anstream",
  "anstyle",
@@ -299,9 +299,9 @@ dependencies = [
 
 [[package]]
 name = "clap_complete"
-version = "4.5.29"
+version = "4.5.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8937760c3f4c60871870b8c3ee5f9b30771f792a7045c48bcbba999d7d6b3b8e"
+checksum = "9646e2e245bf62f45d39a0f3f36f1171ad1ea0d6967fd114bca72cb02a8fcdfb"
 dependencies = [
  "clap",
 ]
@@ -315,7 +315,7 @@ dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
@@ -382,7 +382,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
@@ -393,7 +393,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
 dependencies = [
  "darling_core",
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
@@ -496,9 +496,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
 [[package]]
 name = "futures"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -511,9 +511,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
+checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -521,15 +521,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
+checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d"
+checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -538,27 +538,27 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
+checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
 
 [[package]]
 name = "futures-sink"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
+checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
 
 [[package]]
 name = "futures-task"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
+checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
 
 [[package]]
 name = "futures-util"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
+checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -708,9 +708,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
 
 [[package]]
 name = "js-sys"
-version = "0.3.70"
+version = "0.3.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a"
+checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -780,9 +780,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.159"
+version = "0.2.161"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
+checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1"
 
 [[package]]
 name = "libloading"
@@ -984,12 +984,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.20.1"
+version = "1.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1"
-dependencies = [
- "portable-atomic",
-]
+checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
 
 [[package]]
 name = "pager"
@@ -1092,7 +1089,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
@@ -1130,12 +1127,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
 [[package]]
 name = "prettyplease"
-version = "0.2.22"
+version = "0.2.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba"
+checksum = "910d41a655dac3b764f1ade94821093d3610248694320cd072303a8eedcf221d"
 dependencies = [
  "proc-macro2",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
@@ -1162,18 +1159,18 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.86"
+version = "1.0.88"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
+checksum = "7c3a7fc5db1e57d5a779a352c8cdb57b29aa4c40cc69c3a68a7fedc815fbf2f9"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "pyo3"
-version = "0.22.3"
+version = "0.22.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15ee168e30649f7f234c3d49ef5a7a6cbf5134289bc46c29ff3155fa3221c225"
+checksum = "3d922163ba1f79c04bc49073ba7b32fd5a8d3b76a87c955921234b8e77333c51"
 dependencies = [
  "cfg-if",
  "indoc",
@@ -1190,9 +1187,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.22.3"
+version = "0.22.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e61cef80755fe9e46bb8a0b8f20752ca7676dcc07a5277d8b7768c6172e529b3"
+checksum = "bc38c5feeb496c8321091edf3d63e9a6829eab4b863b4a6a65f26f3e9cc6b179"
 dependencies = [
  "once_cell",
  "target-lexicon",
@@ -1200,9 +1197,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.22.3"
+version = "0.22.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67ce096073ec5405f5ee2b8b31f03a68e02aa10d5d4f565eca04acc41931fa1c"
+checksum = "94845622d88ae274d2729fcefc850e63d7a3ddff5e3ce11bd88486db9f1d357d"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -1210,27 +1207,27 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.22.3"
+version = "0.22.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2440c6d12bc8f3ae39f1e775266fa5122fd0c8891ce7520fa6048e683ad3de28"
+checksum = "e655aad15e09b94ffdb3ce3d217acf652e26bbc37697ef012f5e5e348c716e5e"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.22.3"
+version = "0.22.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1be962f0e06da8f8465729ea2cb71a416d2257dff56cbe40a70d3e62a93ae5d1"
+checksum = "ae1e3f09eecd94618f60a455a23def79f79eba4dc561a97324bf9ac8c6df30ce"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "pyo3-build-config",
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
@@ -1342,7 +1339,7 @@ name = "retis-derive"
 version = "1.4.0"
 dependencies = [
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
@@ -1391,9 +1388,9 @@ dependencies = [
 
 [[package]]
 name = "rustversion"
-version = "1.0.17"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6"
+checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248"
 
 [[package]]
 name = "ryu"
@@ -1403,9 +1400,9 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
 
 [[package]]
 name = "scc"
-version = "2.2.0"
+version = "2.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "836f1e0f4963ef5288b539b643b35e043e76a32d0f4e47e67febf69576527f50"
+checksum = "f2c1f7fc6deb21665a9060dfc7d271be784669295a31babdcd4dd2c79ae8cbfb"
 dependencies = [
  "sdd",
 ]
@@ -1430,14 +1427,14 @@ checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
 name = "sdd"
-version = "3.0.3"
+version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60a7b59a5d9b0099720b417b6325d91a52cbf5b3dcb5041d864be53eefa58abc"
+checksum = "49c1eeaf4b6a87c7479688c6d52b9f1153cedd3c489300564f932b065c6eab95"
 
 [[package]]
 name = "semver"
@@ -1480,14 +1477,14 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.128"
+version = "1.0.132"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
+checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03"
 dependencies = [
  "itoa",
  "memchr",
@@ -1497,9 +1494,9 @@ dependencies = [
 
 [[package]]
 name = "serde_with"
-version = "3.10.0"
+version = "3.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9720086b3357bcb44fce40117d769a4d068c70ecfa190850a980a71755f66fcc"
+checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817"
 dependencies = [
  "base64",
  "chrono",
@@ -1515,14 +1512,14 @@ dependencies = [
 
 [[package]]
 name = "serde_with_macros"
-version = "3.10.0"
+version = "3.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f1abbfe725f27678f4663bcacb75a83e829fd464c25d78dd038a3a29e307cec"
+checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d"
 dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
@@ -1560,7 +1557,7 @@ checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
@@ -1708,9 +1705,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.79"
+version = "2.0.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590"
+checksum = "83540f837a8afc019423a8edb95b52a8effe46957ee402287f4292fae35be021"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1763,7 +1760,7 @@ dependencies = [
  "cfg-if",
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
@@ -1774,7 +1771,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
  "test-case-core",
 ]
 
@@ -1795,7 +1792,7 @@ checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
 ]
 
 [[package]]
@@ -1926,9 +1923,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.93"
+version = "0.2.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
+checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -1937,24 +1934,24 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.93"
+version = "0.2.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b"
+checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358"
 dependencies = [
  "bumpalo",
  "log",
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.93"
+version = "0.2.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf"
+checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -1962,22 +1959,22 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.93"
+version = "0.2.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
+checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.79",
+ "syn 2.0.82",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.93"
+version = "0.2.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
+checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d"
 
 [[package]]
 name = "which"
diff --git a/docs/python.md b/docs/python.md
index 945c6f06f..dfe97486f 100644
--- a/docs/python.md
+++ b/docs/python.md
@@ -6,6 +6,15 @@ python bindings exist to enable writing custom post-processing scripts.
 These bindings can be used in two different ways: the built-in python
 interpreter and the external python library.
 
+## Requirements
+
+Python >= 3.7 is required. In order to build retis on older distros where
+this python version is not available, please run:
+
+```
+$ CARGO_CMD_OPTS="--no-default-features" make
+```
+
 ## Overview
 
 Python bindings currently provide the following basic python classes that
diff --git a/retis/Cargo.toml b/retis/Cargo.toml
index 7d34d1070..518979705 100644
--- a/retis/Cargo.toml
+++ b/retis/Cargo.toml
@@ -19,6 +19,8 @@ release_name = "senyoret"
 maintenance = { status = "actively-developed" }
 
 [features]
+default = ["python"]
+python = ["retis-derive/python", "events/python-embed"]
 test_bindgen_layout = []
 test_cap_bpf = []
 benchmark = []
@@ -33,7 +35,7 @@ caps = "0.5"
 clap = { version = "4.0", features = ["derive", "string"] }
 clap_complete = "4.4"
 elf = "0.7"
-events = {version = "1.4", path = "../retis-events", package="retis-events", features=["python-embed"]}
+events = {version = "1.4", path = "../retis-events", package="retis-events" }
 flate2 = "1.0"
 libbpf-rs = "0.22"
 libbpf-sys = "=1.2"
@@ -49,7 +51,7 @@ plain = "0.2"
 pnet_packet = "0.34"
 rbpf = {version = "0.2", optional = true}
 regex = "1.7"
-retis-derive = {version = "1.4", path = "../retis-derive", features=["python"]}
+retis-derive = {version = "1.4", path = "../retis-derive"}
 serde = {version = "1.0", features = ["derive"]}
 serde_json = "1.0"
 serde_with = "3.0"
diff --git a/retis/src/cli/cli.rs b/retis/src/cli/cli.rs
index 0ad3bfa08..17d5d2806 100644
--- a/retis/src/cli/cli.rs
+++ b/retis/src/cli/cli.rs
@@ -432,6 +432,7 @@ pub(crate) fn get_cli() -> Result<ThinCli> {
     cli.add_subcommand(Box::new(Collect::new()?))?;
     cli.add_subcommand(Box::new(Print::new()?))?;
     cli.add_subcommand(Box::new(Sort::new()?))?;
+    #[cfg(feature = "python")]
     cli.add_subcommand(Box::new(PythonCli::new()?))?;
     cli.add_subcommand(Box::new(Pcap::new()?))?;
     cli.add_subcommand(Box::new(Inspect::new()?))?;
diff --git a/retis/src/process/cli/mod.rs b/retis/src/process/cli/mod.rs
index 1d33bd721..7ca52183f 100644
--- a/retis/src/process/cli/mod.rs
+++ b/retis/src/process/cli/mod.rs
@@ -8,7 +8,9 @@ pub(crate) use self::pcap::*;
 pub(crate) mod print;
 pub(crate) use print::*;
 
+#[cfg(feature = "python")]
 pub(crate) mod python;
+#[cfg(feature = "python")]
 pub(crate) use python::*;
 
 pub(crate) mod sort;