Network exploration and mutation

.gitignore

@@ -5,6 +5,7 @@
 **/*.json
 **/*_binary
 **/*.c
+**/*.html
 
 *.swp
 .hypothesis

lemonspotter/core/function.py

@@ -70,6 +70,13 @@ def return_type(self) -> Type:
 
         return Database().get_type(self._json['return'])
 
+    @property
+    def return_abstract_type(self) -> str:
+        """
+        """
+
+        return self._json['return']
+
     @property  # type: ignore
     @lru_cache()
     def needs_any(self) -> AbstractSet['Function']:

lemonspotter/core/parameter.py

@@ -41,6 +41,13 @@ def type(self) -> Type:
 
         return Database().get_type(self._json['abstract_type'])
 
+    @property
+    def abstract_type(self) -> str:
+        """
+        """
+
+        return self._json['abstract_type']
+
     @property
     def direction(self) -> Direction:
         """This property provides the direction of the Parameter."""

lemonspotter/parsers/standardparser.py

@@ -0,0 +1,81 @@
+"""
+The StandardParser parses the apis.json emitted by the MPI Standard
+pythonization effort.
+"""
+
+
+import json
+import sys
+import logging
+from typing import Mapping, MutableMapping
+
+
+from pathlib import Path
+
+
+from lemonspotter.core.database import Database
+from lemonspotter.core.function import Function
+
+
+class StandardParser:
+    """
+    This class is a parser for the MPI Standard.
+    """
+
+    def __call__(self, database_path: Path) -> None:
+        if database_path.suffix != '.json':
+            raise RuntimeError(f'The given database path is not a json file: '
+                               f'{database_path}')
+
+        with database_path.open() as encoded:
+            apis = json.load(encoded)
+
+            for name, definition in apis.items():
+                if ('lemonspotter' in definition['attributes'] and
+                        definition['attributes']['lemonspotter']):
+                    self._parse_API(definition)
+
+    def _parse_API(self, definition: Mapping) -> None:
+        """
+        Parse the API definition from the MPI Standard.
+        """
+
+        translation = {}
+
+        translation['name'] = definition['name']
+        translation['return'] = definition['return_kind']
+        translation['needs_any'] = definition['attributes']['needs_any']
+        translation['needs_all'] = definition['attributes']['needs_all']
+        translation['leads_any'] = definition['attributes']['leads_any']
+        translation['leads_all'] = definition['attributes']['leads_all']
+        translation['parameters'] = []
+
+        for parameter in definition['parameters']:
+            if parameter['name'] == 'ierror':
+                continue
+
+            param = {}
+
+            param['name'] = parameter['name']
+            param['abstract_type'] = parameter['kind']
+            param['direction'] = parameter['param_direction']
+            param['length'] = parameter['length']
+
+            # TODO pointer
+            # TODO const
+
+            translation['parameters'].append(param)
+
+        # TODO filter, valid combinations of parameters
+
+        Database().add_function(Function(translation))
+
+
+if __name__ == '__main__':
+    parser = StandardParser()
+    parser(Path(sys.argv[1]))
+
+    print(Database()._functions)
+
+    for function in Database()._functions:
+        print(function._json)

lemonspotter/tools/control_dep_graph.py

@@ -0,0 +1,69 @@
+"""
+"""
+
+import sys
+from pathlib import Path
+
+
+from pyvis.network import Network
+
+
+from lemonspotter.parsers.mpiparser import MPIParser
+from lemonspotter.parsers.standardparser import StandardParser
+from lemonspotter.core.database import Database
+
+
+def main() -> None:
+    """
+    """
+
+    # parser = MPIParser()
+    # parser.parse(Path(sys.argv[1]))
+
+    parser = StandardParser()
+    parser(Path(sys.argv[1]))
+
+    net = Network(height='100%', width='100%', directed=True)
+    net.barnes_hut(overlap=1, central_gravity=2)
+
+    # add all nodes
+    for id, function in enumerate(Database().get_functions()):
+        if not function.leads_all and not function.leads_any and not function.needs_all and not function.needs_any:
+            net.add_node(function.name, color='#F4F1BB')
+
+        elif not function.needs_all and not function.needs_any:
+            net.add_node(function.name, color='#ED6A5A')
+
+        elif not function.leads_all and not function.leads_any:
+            net.add_node(function.name, color='#E6EBE0')
+
+        else:
+            net.add_node(function.name)
+
+    # add need edges
+    for function in Database().get_functions():
+        for need in function.needs_any:
+            if function in need.leads_all or function in need.leads_any:
+                continue
+
+            net.add_edge(need.name, function.name, color='#95C623')
+
+    # add lead edges
+    for function in Database().get_functions():
+        for lead in function.leads_any:
+            if function in lead.needs_all or function in lead.needs_any:
+                continue
+
+            net.add_edge(function.name, lead.name, color='#0E4749')
+
+    # add critical edges, start -> end
+    for function in Database().get_functions():
+        for lead in function.leads_any:
+            if function in lead.needs_any:
+                net.add_edge(function.name, lead.name, color='#E55812')
+
+    net.show('plot.html')
+
+
+if __name__ == '__main__':
+    main()

lemonspotter/tools/control_flow_graphs.py

@@ -0,0 +1,110 @@
+"""
+"""
+
+import sys
+from pathlib import Path
+from random import choice, random
+from itertools import chain
+
+
+from pyvis.network import Network
+
+
+from lemonspotter.parsers.mpiparser import MPIParser
+from lemonspotter.parsers.standardparser import StandardParser
+from lemonspotter.core.database import Database
+
+
+def main() -> None:
+    """
+    """
+
+    # parser = MPIParser()
+    # parser.parse(Path(sys.argv[1]))
+
+    parser = StandardParser()
+    parser(Path(sys.argv[1]))
+
+    net = Network(height='100%', width='100%', directed=True, layout=True)
+    # net.barnes_hut(overlap=1, central_gravity=2)
+
+    nodes = Database().get_functions()
+
+    # # filter down to non-independent
+    # # TODO ignore for now, include after, they can be used always
+    # nodes = [node for node in nodes if (node.needs_all or node.needs_any or
+    #                                     node.leads_any or node.leads_all)]
+
+    print(nodes)
+
+    # starting points
+    no_need_nodes = [node for node in nodes if (not node.needs_any and not node.needs_all and
+                                                (node.leads_any or node.leads_all))]
+    print(no_need_nodes)
+
+    # end points
+    no_lead_nodes = [node for node in nodes if (not node.leads_any and not node.leads_all and
+                                                (node.needs_any or node.needs_all))]
+    print(no_lead_nodes)
+
+    internal_nodes = [node for node in nodes if ((node.leads_all or node.leads_any) and
+                                                 (node.needs_all or node.needs_any))]
+    print(internal_nodes)
+
+    independent_nodes = [node for node in nodes if (not node.leads_all and not node.leads_any and
+                                                    not node.needs_all and not node.needs_any)]
+    print(independent_nodes)
+
+    paths = []
+
+    def generate_path():
+        path = []
+
+        # start
+        path.append(no_need_nodes[0])
+
+        while random() > 0.01:
+            node = choice(list(chain(independent_nodes, internal_nodes)))
+
+            if node in path:
+                continue
+
+            path.append(node)
+
+        # end
+        path.append(no_lead_nodes[0])
+
+        return path
+
+    for idx in range(1000):
+        path = generate_path()
+
+        if path not in paths:
+            paths.append(path)
+
+    for idx, path in enumerate(paths):
+        # nodes
+        for node in path:
+            if node in no_lead_nodes:
+                # end
+                net.add_node(node.name+'_'+str(idx), label=node.name, color='#E76F51')
+
+            elif node in no_need_nodes:
+                # start
+                net.add_node(node.name+'_'+str(idx), label=node.name, color='#F4A261')
+
+            elif node in independent_nodes:
+                net.add_node(node.name+'_'+str(idx), label=node.name, color='#E9C46A')
+
+            else:
+                net.add_node(node.name+'_'+str(idx), label=node.name, color='#2A9D8F')
+
+        # edges
+        for n0, n1 in zip(path, path[1:]):
+            net.add_edge(n0.name+'_'+str(idx), n1.name+'_'+str(idx), color='#264653')
+
+    net.show('plot.html')
+
+
+if __name__ == '__main__':
+    main()

lemonspotter/tools/data_dep_graph.py

@@ -0,0 +1,75 @@
+"""
+"""
+
+import sys
+from pathlib import Path
+
+
+from pyvis.network import Network
+
+
+from lemonspotter.parsers.mpiparser import MPIParser
+from lemonspotter.parsers.standardparser import StandardParser
+from lemonspotter.core.database import Database
+from lemonspotter.core.parameter import Direction
+
+
+def main() -> None:
+    """
+    """
+
+    # parser = MPIParser()
+    # parser.parse(Path(sys.argv[1]))
+
+    parser = StandardParser()
+    parser(Path(sys.argv[1]))
+
+    net = Network(height='100%', width='100%', directed=True)
+    net.barnes_hut(overlap=1, central_gravity=2)
+
+    # add all constants
+    # net.add_node('MPI_COMM_WORLD', color='#F4A261')
+    # net.add_node('MPI_COMM_SELF', color='#F4A261')
+    # net.add_node('MPI_COMM_NULL', color='#F4A261')
+
+    # add all nodes
+    for id, function in enumerate(Database().get_functions()):
+        net.add_node(function.name)
+
+    # draw edges between constructors to consumers
+
+    for producer in Database().get_functions():
+        outs = []
+        outs.append(producer.return_abstract_type)
+        outs.extend([param.abstract_type for param in producer.parameters if param.direction == Direction('out')])
+
+        for out_type in outs:
+            found = False
+
+            for consumer in Database().get_functions():
+                # TODO self loop allowed?
+
+                for param in consumer.parameters:
+                    if param.direction == Direction('in') and param.abstract_type == out_type:
+                        # draw edge
+                        net.add_edge(producer.name, consumer.name, title=out_type)
+                        found = True
+
+            if not found:
+                if not ('TYPE_' + out_type) in net.get_nodes():
+                    net.add_node('TYPE_' + out_type, color='#E76F51')
+
+                net.add_edge(producer.name, 'TYPE_' + out_type, color='#E76F51')
+
+        for in_type in [param.abstract_type for param in producer.parameters if param.direction == Direction('in')]:
+            if in_type not in outs:
+                net.add_node('TYPE_' + in_type, color='#2A9D8F')
+                net.add_edge('TYPE_' + in_type, producer.name, color='#2A9D8F')
+
+    # constants exist, how do we represent it? a node per constant or a single constant node?
+
+    net.show('plot.html')
+
+
+if __name__ == '__main__':
+    main()