From f90f5670a0a88f57ee9687913781578df65d0000 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Tue, 10 Sep 2024 16:50:31 -0400 Subject: [PATCH 1/2] Add example of using scrape on the primary property (#3907) --- changes/c8bc54e714df2448fe9b1cd440f3eba3.yaml | 5 +++++ synapse/lib/storm.py | 3 +++ 2 files changed, 8 insertions(+) create mode 100644 changes/c8bc54e714df2448fe9b1cd440f3eba3.yaml diff --git a/changes/c8bc54e714df2448fe9b1cd440f3eba3.yaml b/changes/c8bc54e714df2448fe9b1cd440f3eba3.yaml new file mode 100644 index 0000000000..3709d9c068 --- /dev/null +++ b/changes/c8bc54e714df2448fe9b1cd440f3eba3.yaml @@ -0,0 +1,5 @@ +--- +desc: Added an example of using ``scrape`` on the primary property to the command usage statement. +prs: [] +type: doc +... diff --git a/synapse/lib/storm.py b/synapse/lib/storm.py index 6073713b5e..7e03c9237a 100644 --- a/synapse/lib/storm.py +++ b/synapse/lib/storm.py @@ -5505,6 +5505,9 @@ class ScrapeCmd(Cmd): # Scrape only the :engine and :text props from the inbound nodes. inet:search:query | scrape :text :engine + # Scrape the primary property from the inbound nodes. + it:dev:str | scrape $node.repr() + # Scrape properties inbound nodes and yield newly scraped nodes. inet:search:query | scrape --yield From 13693123ee5a40ba676bad88ad3ff5be58082849 Mon Sep 17 00:00:00 2001 From: vEpiphyte Date: Tue, 10 Sep 2024 17:01:58 -0400 Subject: [PATCH 2/2] Add inet:http:resp history support (SYN-2307) (#3900) --- changes/aa85cce4417ce55f01a884c4ab464aa4.yaml | 5 ++ synapse/lib/stormhttp.py | 30 +++++++++- synapse/tests/test_lib_stormhttp.py | 59 ++++++++++++++++++- 3 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 changes/aa85cce4417ce55f01a884c4ab464aa4.yaml diff --git a/changes/aa85cce4417ce55f01a884c4ab464aa4.yaml b/changes/aa85cce4417ce55f01a884c4ab464aa4.yaml new file mode 100644 index 0000000000..40e543bbbd --- /dev/null +++ b/changes/aa85cce4417ce55f01a884c4ab464aa4.yaml @@ -0,0 +1,5 @@ +--- +desc: Add request history on the Storm ``inet:http:resp`` object. +prs: [] +type: feat +... diff --git a/synapse/lib/stormhttp.py b/synapse/lib/stormhttp.py index 88e80d5b5c..13f751495f 100644 --- a/synapse/lib/stormhttp.py +++ b/synapse/lib/stormhttp.py @@ -464,12 +464,27 @@ async def _httpRequest(self, meth, url, headers=None, json=None, body=None, kwargs['json'] = json async with sess.request(meth, url, headers=headers, **kwargs) as resp: + history = [] + for hist in resp.history: + hnfo = { + 'code': hist.status, + 'reason': await self.codereason(hist.status), + 'headers': dict(hist.headers), + 'url': str(hist.url), + # aiohttp has already closed the connection by this point + # so there is no connection to read a body from. + 'body': b'', + 'history': [], + 'request_headers': dict(hist.request_info.headers) + } + history.append(hnfo) info = { 'code': resp.status, 'reason': await self.codereason(resp.status), 'headers': dict(resp.headers), 'url': str(resp.url), 'body': await resp.read(), + 'history': history, 'request_headers': dict(resp.request_info.headers) } return HttpResp(info) @@ -486,13 +501,14 @@ async def _httpRequest(self, meth, url, headers=None, json=None, body=None, reason = f'Exception occurred during request: {err[0]}' info = { + 'err': err, 'code': -1, 'reason': reason, 'headers': dict(), - 'request_headers': dict(), 'url': url, 'body': b'', - 'err': err, + 'history': [], + 'request_headers': dict(), } return HttpResp(info) @@ -511,6 +527,9 @@ class HttpResp(s_stormtypes.Prim): {'name': 'url', 'type': 'str', 'desc': 'The response URL. If the request was redirected, this would be the final URL in the redirection chain. If the status code is -1, then this is the request URL.'}, {'name': 'err', 'type': 'list', 'desc': 'Tuple of the error type and information if an exception occurred.'}, + {'name': 'history', 'desc': 'A list of response objects representing the history of the response. This is populated when responses are redirected.', + 'type': {'type': 'gtor', '_gtorfunc': '_gtorHistory', + 'returns': {'type': 'list', 'desc': 'A list of ``inet:http:resp`` objects.', }}}, {'name': 'json', 'desc': 'Get the JSON deserialized response.', 'type': {'type': 'function', '_funcname': '_httpRespJson', 'args': ( @@ -538,6 +557,10 @@ def __init__(self, valu, path=None): self.locls['request_headers'] = self.valu.get('request_headers') self.locls['err'] = self.valu.get('err', ()) + self.gtors.update({ + 'history': self._gtorHistory, + }) + def getObjLocals(self): return { 'json': self._httpRespJson, @@ -568,3 +591,6 @@ async def _httpRespMsgpack(self): unpk = s_msgpack.Unpk() for _, item in unpk.feed(byts): yield item + + async def _gtorHistory(self): + return [HttpResp(hnfo) for hnfo in self.valu.get('history')] diff --git a/synapse/tests/test_lib_stormhttp.py b/synapse/tests/test_lib_stormhttp.py index 51e10f18ec..1af920d325 100644 --- a/synapse/tests/test_lib_stormhttp.py +++ b/synapse/tests/test_lib_stormhttp.py @@ -85,7 +85,6 @@ async def test_storm_http_get(self): q = ''' $_url = `https://root:root@127.0.0.1:{($port + (1))}/api/v0/newp` $resp = $lib.inet.http.get($_url, ssl_verify=$lib.false) - $lib.log.info(`{$resp}`) if ( $resp.code != (-1) ) { $lib.exit(mesg='Test fail!') } return ( $resp.url ) ''' @@ -216,6 +215,64 @@ async def test_storm_http_get(self): retn = await core.callStorm('return($lib.inet.http.codereason(123))') self.eq(retn, 'Unknown HTTP status code 123') + # Request history is preserved across multiple redirects. + q = '''$api = $lib.cortex.httpapi.add(dyn00) + $api.methods.get = ${ + $api = $request.api + if $n { + $part = `redir0{$n}` + $redir = `/api/ext/{$part}` + $headers = ({"Location": $redir}) + $api.vars.n = ($n - (1) ) + $api.path = $part + $request.reply(301, headers=$headers) + } else { + $api.vars.n = $initial_n + $api.path = dyn00 + $request.reply(200, body=({"end": "youMadeIt"}) ) + } + } + $api.authenticated = (false) + $api.vars.initial_n = (3) + $api.vars.n = (3) + return ( ($api.iden) )''' + iden00 = await core.callStorm(q) + + q = ''' + $url = `https://127.0.0.1:{$port}/api/ext/dyn00` + $resp = $lib.inet.http.get($url, ssl_verify=$lib.false) + return ( $resp ) + ''' + resp = await core.callStorm(q, opts=opts) + self.eq(resp.get('url'), f'https://127.0.0.1:{port}/api/ext/redir01') + self.eq([hnfo.get('url') for hnfo in resp.get('history')], + [f'https://127.0.0.1:{port}/api/ext/dyn00', + f'https://127.0.0.1:{port}/api/ext/redir03', + f'https://127.0.0.1:{port}/api/ext/redir02', + ]) + + # The gtor returns a list of objects + q = ''' + $url = `https://127.0.0.1:{$port}/api/ext/dyn00` + $resp = $lib.inet.http.get($url, ssl_verify=$lib.false) + return ( $resp.history.0 ) + ''' + resp = await core.callStorm(q, opts=opts) + self.eq(resp.get('url'), f'https://127.0.0.1:{port}/api/ext/dyn00') + + # The history is not available if there is a fatal error when + # following redirects. + q = ''' + $_url = `https://127.0.0.1:{($port + (1))}/api/v0/newp` + $params = ({'redirect': $_url}) + $resp = $lib.inet.http.get($url, params=$params, ssl_verify=$lib.false) + if ( $resp.code != (-1) ) { $lib.exit(mesg='Test fail!') } + return ( $resp.history ) + ''' + resp = await core.callStorm(q, opts=opts) + self.isinstance(resp, tuple) + self.len(0, resp) + async def test_storm_http_inject_ca(self): with self.getTestDir() as dirn: