iproute: draft route_dump() & route_dumps()

A proof of concept of a binary stream dumper.

* ipr.route_dump(fd, family=AF_UNSPEC) -- save routes to a
  file object; it can be an open file, BytesIO() or like that
* ipr.route_dumps(family=AF_UNSPEC) -- return routes as a binary
  string

Bug-Url: #1203

pyroute2/iproute/linux.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
+import io
 import logging
 import os
+import struct
 import time
 import warnings
 from functools import partial
@@ -91,9 +93,10 @@
 from pyroute2.requests.rule import RuleFieldFilter, RuleIPRouteFilter
 from pyroute2.requests.tc import TcIPRouteFilter, TcRequestFilter
 
-from .parsers import default_routes
+from .parsers import default_routes, export_routes
 
 DEFAULT_TABLE = 254
+IPROUTE2_DUMP_MAGIC = 0x45311224
 log = logging.getLogger(__name__)
 
 
@@ -405,6 +408,30 @@ async def probe(self, command, **kwarg):
         await request.send()
         return [x async for x in request.response()]
 
+    # 8<---------------------------------------------------------------
+    #
+    # Binary streams methods
+    #
+    async def route_dump(self, fd, family=AF_UNSPEC, fmt='iproute2'):
+        if fmt == 'iproute2':
+            fd.write(struct.pack('I', IPROUTE2_DUMP_MAGIC))
+        msg = rtmsg()
+        msg['family'] = family
+        request = NetlinkRequest(
+            self,
+            msg,
+            msg_type=RTM_GETROUTE,
+            msg_flags=NLM_F_DUMP | NLM_F_REQUEST,
+            parser=export_routes(fd),
+        )
+        await request.send()
+        return [x async for x in request.response()]
+
+    async def route_dumps(self, family=AF_UNSPEC, fmt='iproute2'):
+        fd = io.BytesIO()
+        await self.route_dump(fd, family, fmt)
+        return fd.getvalue()
+
     # 8<---------------------------------------------------------------
     #
     # Listing methods
@@ -2458,6 +2485,8 @@ def __getattr__(self, name):
             'flush_rules',
             'flush_routes',
             'get_netnsid',
+            'route_dump',
+            'route_dumps',
         ]
         async_dump_methods = [
             'dump',

pyroute2/iproute/parsers.py

@@ -1,9 +1,49 @@
 import struct
+from functools import partial
 
 from pyroute2.netlink import NLMSG_DONE, nlmsg
+from pyroute2.netlink.exceptions import NetlinkError
+from pyroute2.netlink.rtnl import RTM_NEWROUTE
 from pyroute2.netlink.rtnl.rtmsg import rtmsg
 
 
+def get_header(data, offset):
+    # get message header
+    header = dict(
+        zip(
+            ('length', 'type', 'flags', 'sequence_number'),
+            struct.unpack_from('IHHI', data, offset),
+        )
+    )
+    header['error'] = None
+    return header
+
+
+def msg_done(header):
+    msg = nlmsg()
+    msg['header'] = header
+    msg.length = msg['header']['length']
+    return msg
+
+
+def _export_routes(fd, data, offset, length):
+    '''Export RTM_NEWROUTE messages binary data.
+
+    Otherwise return NLMSG_DONE.
+    '''
+    header = get_header(data, offset)
+    if header['type'] == NLMSG_DONE:
+        return msg_done(header)
+    elif header['type'] == RTM_NEWROUTE:
+        fd.write(data[offset : offset + length])
+        return
+    raise NetlinkError()
+
+
+def export_routes(fd):
+    return partial(_export_routes, fd)
+
+
 def default_routes(data, offset, length):
     '''
     Only for RTM_NEWROUTE.
@@ -14,19 +54,9 @@ def default_routes(data, offset, length):
     * nlmsg() -- NLMSG_DONE
     * None for any other messages
     '''
-    # get message header
-    header = dict(
-        zip(
-            ('length', 'type', 'flags', 'sequence_number'),
-            struct.unpack_from('IHHI', data, offset),
-        )
-    )
-    header['error'] = None
+    header = get_header(data, offset)
     if header['type'] == NLMSG_DONE:
-        msg = nlmsg()
-        msg['header'] = header
-        msg.length = msg['header']['length']
-        return msg
+        return msg_done(header)
 
     # skip to NLA: offset + nlmsg header + rtmsg data
     cursor = offset + 28

tests/test_core/test_ipr/test_route_async.py

@@ -1,8 +1,78 @@
+import io
+from socket import AF_INET, AF_INET6, AF_UNSPEC
+
 import pytest
 
 from pyroute2 import AsyncIPRoute
 
 
+@pytest.mark.parametrize(
+    'family,target_tables,target_families,fmt,offset',
+    [
+        (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, 'iproute2', 4),
+        (AF_INET, {254, 255}, {AF_INET}, 'iproute2', 4),
+        (AF_INET6, {254, 255}, {AF_INET6}, 'iproute2', 4),
+        (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, None, 0),
+        (AF_INET, {254, 255}, {AF_INET}, None, 0),
+        (AF_INET6, {254, 255}, {AF_INET6}, None, 0),
+    ],
+    ids=(
+        'iproute2/AF_UNSPEC',
+        'iproute2/AF_INET',
+        'iproute2/AF_INET6',
+        'native/AF_UNSPEC',
+        'native/AF_INET',
+        'native/AF_INET6',
+    ),
+)
+@pytest.mark.asyncio
+async def test_route_dump(
+    async_ipr, family, target_tables, target_families, fmt, offset
+):
+    fd = io.BytesIO()
+    await async_ipr.route_dump(fd, family=family, fmt=fmt)
+    tables = set()
+    families = set()
+    for route in async_ipr.marshal.parse(fd.getvalue()[offset:]):
+        tables.add(route.get('table'))
+        families.add(route.get('family'))
+    assert tables >= target_tables
+    assert families == target_families
+
+
+@pytest.mark.parametrize(
+    'family,target_tables,target_families,fmt,offset',
+    [
+        (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, 'iproute2', 4),
+        (AF_INET, {254, 255}, {AF_INET}, 'iproute2', 4),
+        (AF_INET6, {254, 255}, {AF_INET6}, 'iproute2', 4),
+        (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, None, 0),
+        (AF_INET, {254, 255}, {AF_INET}, None, 0),
+        (AF_INET6, {254, 255}, {AF_INET6}, None, 0),
+    ],
+    ids=(
+        'iproute2/AF_UNSPEC',
+        'iproute2/AF_INET',
+        'iproute2/AF_INET6',
+        'native/AF_UNSPEC',
+        'native/AF_INET',
+        'native/AF_INET6',
+    ),
+)
+@pytest.mark.asyncio
+async def test_route_dumps(
+    async_ipr, family, target_tables, target_families, fmt, offset
+):
+    data = await async_ipr.route_dumps(family=family, fmt=fmt)
+    tables = set()
+    families = set()
+    for route in async_ipr.marshal.parse(data[offset:]):
+        tables.add(route.get('table'))
+        families.add(route.get('family'))
+    assert tables >= target_tables
+    assert families == target_families
+
+
 @pytest.mark.parametrize(
     "command,kwarg",
     [

tests/test_core/test_ipr/test_route_sync.py

@@ -1,8 +1,76 @@
+import io
+from socket import AF_INET, AF_INET6, AF_UNSPEC
+
 import pytest
 
 from pyroute2 import IPRoute
 
 
+@pytest.mark.parametrize(
+    'family,target_tables,target_families,fmt,offset',
+    [
+        (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, 'iproute2', 4),
+        (AF_INET, {254, 255}, {AF_INET}, 'iproute2', 4),
+        (AF_INET6, {254, 255}, {AF_INET6}, 'iproute2', 4),
+        (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, None, 0),
+        (AF_INET, {254, 255}, {AF_INET}, None, 0),
+        (AF_INET6, {254, 255}, {AF_INET6}, None, 0),
+    ],
+    ids=(
+        'iproute2/AF_UNSPEC',
+        'iproute2/AF_INET',
+        'iproute2/AF_INET6',
+        'native/AF_UNSPEC',
+        'native/AF_INET',
+        'native/AF_INET6',
+    ),
+)
+def test_route_dump(
+    sync_ipr, family, target_tables, target_families, fmt, offset
+):
+    fd = io.BytesIO()
+    sync_ipr.route_dump(fd, family=family, fmt=fmt)
+    tables = set()
+    families = set()
+    for route in sync_ipr.marshal.parse(fd.getvalue()[offset:]):
+        tables.add(route.get('table'))
+        families.add(route.get('family'))
+    assert tables >= target_tables
+    assert families == target_families
+
+
+@pytest.mark.parametrize(
+    'family,target_tables,target_families,fmt,offset',
+    [
+        (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, 'iproute2', 4),
+        (AF_INET, {254, 255}, {AF_INET}, 'iproute2', 4),
+        (AF_INET6, {254, 255}, {AF_INET6}, 'iproute2', 4),
+        (AF_UNSPEC, {254, 255}, {AF_INET, AF_INET6}, None, 0),
+        (AF_INET, {254, 255}, {AF_INET}, None, 0),
+        (AF_INET6, {254, 255}, {AF_INET6}, None, 0),
+    ],
+    ids=(
+        'iproute2/AF_UNSPEC',
+        'iproute2/AF_INET',
+        'iproute2/AF_INET6',
+        'native/AF_UNSPEC',
+        'native/AF_INET',
+        'native/AF_INET6',
+    ),
+)
+def test_route_dumps(
+    sync_ipr, family, target_tables, target_families, fmt, offset
+):
+    data = sync_ipr.route_dumps(family=family, fmt=fmt)
+    tables = set()
+    families = set()
+    for route in sync_ipr.marshal.parse(data[offset:]):
+        tables.add(route.get('table'))
+        families.add(route.get('family'))
+    assert tables >= target_tables
+    assert families == target_families
+
+
 @pytest.mark.parametrize(
     "command,kwarg",
     [