Merge branch 'master' into visi-algo-tweak

changes/aa85cce4417ce55f01a884c4ab464aa4.yaml

@@ -0,0 +1,5 @@
+---
+desc: Add request history on the Storm ``inet:http:resp`` object.
+prs: []
+type: feat
+...

changes/c8bc54e714df2448fe9b1cd440f3eba3.yaml

@@ -0,0 +1,5 @@
+---
+desc: Added an example of using ``scrape`` on the primary property to the command usage statement.
+prs: []
+type: doc
+...

synapse/lib/storm.py

@@ -5505,6 +5505,9 @@ class ScrapeCmd(Cmd):
         # Scrape only the :engine and :text props from the inbound nodes.
         inet:search:query | scrape :text :engine
 
+        # Scrape the primary property from the inbound nodes.
+        it:dev:str | scrape $node.repr()
+
         # Scrape properties inbound nodes and yield newly scraped nodes.
         inet:search:query | scrape --yield
 

synapse/lib/stormhttp.py

@@ -464,12 +464,27 @@ async def _httpRequest(self, meth, url, headers=None, json=None, body=None,
                     kwargs['json'] = json
 
                 async with sess.request(meth, url, headers=headers, **kwargs) as resp:
+                    history = []
+                    for hist in resp.history:
+                        hnfo = {
+                            'code': hist.status,
+                            'reason': await self.codereason(hist.status),
+                            'headers': dict(hist.headers),
+                            'url': str(hist.url),
+                            # aiohttp has already closed the connection by this point
+                            # so there is no connection to read a body from.
+                            'body': b'',
+                            'history': [],
+                            'request_headers': dict(hist.request_info.headers)
+                        }
+                        history.append(hnfo)
                     info = {
                         'code': resp.status,
                         'reason': await self.codereason(resp.status),
                         'headers': dict(resp.headers),
                         'url': str(resp.url),
                         'body': await resp.read(),
+                        'history': history,
                         'request_headers': dict(resp.request_info.headers)
                     }
                     return HttpResp(info)
@@ -486,13 +501,14 @@ async def _httpRequest(self, meth, url, headers=None, json=None, body=None,
                     reason = f'Exception occurred during request: {err[0]}'
 
                 info = {
+                    'err': err,
                     'code': -1,
                     'reason': reason,
                     'headers': dict(),
-                    'request_headers': dict(),
                     'url': url,
                     'body': b'',
-                    'err': err,
+                    'history': [],
+                    'request_headers': dict(),
                 }
                 return HttpResp(info)
 
@@ -511,6 +527,9 @@ class HttpResp(s_stormtypes.Prim):
         {'name': 'url', 'type': 'str',
          'desc': 'The response URL. If the request was redirected, this would be the final URL in the redirection chain. If the status code is -1, then this is the request URL.'},
         {'name': 'err', 'type': 'list', 'desc': 'Tuple of the error type and information if an exception occurred.'},
+        {'name': 'history', 'desc': 'A list of response objects representing the history of the response. This is populated when responses are redirected.',
+         'type': {'type': 'gtor', '_gtorfunc': '_gtorHistory',
+                  'returns': {'type': 'list', 'desc': 'A list of ``inet:http:resp`` objects.', }}},
         {'name': 'json', 'desc': 'Get the JSON deserialized response.',
          'type': {'type': 'function', '_funcname': '_httpRespJson',
                   'args': (
@@ -538,6 +557,10 @@ def __init__(self, valu, path=None):
         self.locls['request_headers'] = self.valu.get('request_headers')
         self.locls['err'] = self.valu.get('err', ())
 
+        self.gtors.update({
+            'history': self._gtorHistory,
+        })
+
     def getObjLocals(self):
         return {
             'json': self._httpRespJson,
@@ -568,3 +591,6 @@ async def _httpRespMsgpack(self):
         unpk = s_msgpack.Unpk()
         for _, item in unpk.feed(byts):
             yield item
+
+    async def _gtorHistory(self):
+        return [HttpResp(hnfo) for hnfo in self.valu.get('history')]

synapse/tests/test_lib_stormhttp.py

@@ -85,7 +85,6 @@ async def test_storm_http_get(self):
             q = '''
             $_url = `https://root:root@127.0.0.1:{($port + (1))}/api/v0/newp`
             $resp = $lib.inet.http.get($_url, ssl_verify=$lib.false)
-            $lib.log.info(`{$resp}`)
             if ( $resp.code != (-1) ) { $lib.exit(mesg='Test fail!') }
             return ( $resp.url )
             '''
@@ -216,6 +215,64 @@ async def test_storm_http_get(self):
             retn = await core.callStorm('return($lib.inet.http.codereason(123))')
             self.eq(retn, 'Unknown HTTP status code 123')
 
+            # Request history is preserved across multiple redirects.
+            q = '''$api = $lib.cortex.httpapi.add(dyn00)
+            $api.methods.get =  ${
+                $api = $request.api
+                if $n {
+                    $part = `redir0{$n}`
+                    $redir = `/api/ext/{$part}`
+                    $headers = ({"Location": $redir})
+                    $api.vars.n = ($n - (1) )
+                    $api.path = $part
+                    $request.reply(301, headers=$headers)
+                } else {
+                    $api.vars.n = $initial_n
+                    $api.path = dyn00
+                    $request.reply(200, body=({"end": "youMadeIt"}) )
+                }
+            }
+            $api.authenticated = (false)
+            $api.vars.initial_n = (3)
+            $api.vars.n = (3)
+            return ( ($api.iden) )'''
+            iden00 = await core.callStorm(q)
+
+            q = '''
+            $url = `https://127.0.0.1:{$port}/api/ext/dyn00`
+            $resp = $lib.inet.http.get($url, ssl_verify=$lib.false)
+            return ( $resp )
+            '''
+            resp = await core.callStorm(q, opts=opts)
+            self.eq(resp.get('url'), f'https://127.0.0.1:{port}/api/ext/redir01')
+            self.eq([hnfo.get('url') for hnfo in resp.get('history')],
+                    [f'https://127.0.0.1:{port}/api/ext/dyn00',
+                     f'https://127.0.0.1:{port}/api/ext/redir03',
+                     f'https://127.0.0.1:{port}/api/ext/redir02',
+                     ])
+
+            # The gtor returns a list of objects
+            q = '''
+            $url = `https://127.0.0.1:{$port}/api/ext/dyn00`
+            $resp = $lib.inet.http.get($url, ssl_verify=$lib.false)
+            return ( $resp.history.0 )
+            '''
+            resp = await core.callStorm(q, opts=opts)
+            self.eq(resp.get('url'), f'https://127.0.0.1:{port}/api/ext/dyn00')
+
+            # The history is not available if there is a fatal error when
+            # following redirects.
+            q = '''
+            $_url = `https://127.0.0.1:{($port + (1))}/api/v0/newp`
+            $params = ({'redirect': $_url})
+            $resp = $lib.inet.http.get($url, params=$params, ssl_verify=$lib.false)
+            if ( $resp.code != (-1) ) { $lib.exit(mesg='Test fail!') }
+            return ( $resp.history )
+            '''
+            resp = await core.callStorm(q, opts=opts)
+            self.isinstance(resp, tuple)
+            self.len(0, resp)
+
     async def test_storm_http_inject_ca(self):
 
         with self.getTestDir() as dirn: