lunaix-os/scripts/expand.py

   1 import jinja2
   2 import re
   3 import argparse
   4 import math
   5 import json
   6 from pathlib import Path
   7 from abc import ABC, abstractmethod
   8
   9 class Preprocessor:
  10     reHex = re.compile(r"^0x([0-9a-fA-F]+)$")
  11     reGranuel = re.compile(r"^(?P<num>[0-9]+)@(?P<g>.+)$")
  12     reMacroRef = re.compile(r"^\*(?P<var>[a-zA-z0-9]+)$")
  13     reInt = re.compile(r"^[0-9]+$")
  14     def __init__(self) -> None:
  15         pass
  16
  17     @staticmethod
  18     def expand_str(s: str, param_dict):
  19         if Preprocessor.reInt.match(s) is not None:
  20             return int(s)
  21
  22         mo = Preprocessor.reHex.match(s)
  23         if mo is not None:
  24             return int(s, 16)
  25
  26         mo = Preprocessor.reGranuel.match(s)
  27         if mo is not None:
  28             mg = mo.groupdict()
  29             num = int(mg['num'])
  30             granuel = param_dict["granule"][mg['g']]
  31             return num * granuel
  32
  33         mo = Preprocessor.reMacroRef.match(s)
  34         if mo is not None:
  35             mg = mo.groupdict()
  36             return param_dict[mg['var']]
  37
  38         return s.format_map(param_dict)
  39
  40 class DataObject(ABC):
  41     def __init__(self, name, record):
  42         self.key = name
  43         self._record = record
  44         self.user_field = {}
  45         self.ctrl_field = {}
  46         self._parse(record)
  47
  48     @staticmethod
  49     def create(record):
  50         return DataObject.create("", record)
  51
  52     @staticmethod
  53     def create(key, record):
  54         if PrimitiveType.can_create(record):
  55             return PrimitiveType(record)
  56
  57         name = key
  58         t = name if "$type" not in record else record['$type']
  59
  60         if "$name" in record:
  61             name = record["$name"].strip()
  62
  63         if not key.startswith('@') and "$type" not in record:
  64             return PlainOldObject(name, record)
  65
  66         t = t.strip("@")
  67         if t == "list":
  68             return RangedObject(name, record)
  69         elif t == "foreach":
  70             return ForEachIndexObject(name, record)
  71         elif t == "case_range_index":
  72             return Condition(record)
  73         elif t == "data":
  74             return PlainOldObject(name, record)
  75         elif t == "define":
  76             return VariableDeclarationObject(record)
  77         elif t == "memory_map":
  78             return MemoryMapObject(record)
  79         else:
  80             return RawObject(name, record)
  81
  82     def _parse(self, record):
  83         for k, v in record.items():
  84             if k.startswith("$"):
  85                 self.ctrl_field[k.strip("$")] = FieldType.create(k, v)
  86             elif k.startswith("@"):
  87                 self.ctrl_field[k.strip("@")] = DataObject.create(k, v)
  88             else:
  89                 self.user_field[k] = DataObject.create(k, v)
  90
  91     def expand(self, param={}):
  92         obj2 = {}
  93         for f in self.ctrl_field.values():
  94             if not isinstance(f, DataObject):
  95                 continue
  96             obj2.update(**f.expand(param))
  97
  98         obj = {}
  99         _param = {**param, **obj2}
 100         for k, v in self.user_field.items():
 101             if isinstance(v, DataObject):
 102                 obj[k] = v.expand(_param)
 103             else:
 104                 obj[k] = v
 105
 106         return {**obj, **obj2}
 107
 108
 109 class FieldType:
 110     def __init__(self, record) -> None:
 111         self._record = record
 112         self._parse(record)
 113
 114     @staticmethod
 115     def create(field, value):
 116         if field == "$range":
 117             return RangeType(value)
 118         else:
 119             return value
 120
 121     @abstractmethod
 122     def _parse(self, record):
 123         pass
 124
 125     @abstractmethod
 126     def get(self, param):
 127         pass
 128
 129     def getraw(self):
 130         return self.__record
 131
 132 class PrimitiveType(DataObject):
 133     def __init__(self, record) -> None:
 134         super().__init__("", record)
 135
 136     @staticmethod
 137     def can_create(value):
 138         return type(value) in (str, int, bool)
 139
 140     def _parse(self, record):
 141         if type(record) not in (str, int, bool):
 142             raise Exception(f"{type(self).__name__} require primitive type input")
 143         self.val = record
 144
 145         if type(record) == str:
 146             self.__get_fn = self.__process_str
 147         else:
 148             self.__get_fn = lambda x: self.val
 149
 150     def __process_str(self, param):
 151         return Preprocessor.expand_str(self.val, param)
 152
 153     def expand(self, param={}):
 154         return self.__get_fn(param)
 155
 156 class RangeType(FieldType):
 157     def __init__(self, record) -> None:
 158         self.__ranged_component = re.compile(r"^(?P<index>[^.]+)$|^(?P<start>.+?)\.\.(?P<end>.+)$")
 159         super().__init__(record)
 160
 161     def _parse(self, record):
 162         return super()._parse(record)
 163
 164     def get(self, param):
 165         record = self._record.strip('[]')
 166
 167         self.__value=[]
 168         for component in record.split(','):
 169             component = component.strip()
 170             mo = self.__ranged_component.match(component)
 171             if mo is None:
 172                 raise Exception(f"value '{component}' is not valid range component")
 173
 174             mo = mo.groupdict()
 175             if mo["index"] is not None:
 176                 self.__value.append(Preprocessor.expand_str(mo['index'], param))
 177             else:
 178                 start = Preprocessor.expand_str(mo['start'], param)
 179                 end = Preprocessor.expand_str(mo['end'], param)
 180                 self.__value += [x for x in range(start, end + 1)]
 181         return self.__value
 182
 183     def getraw(self):
 184         return self._record
 185
 186 class VariableDeclarationObject(DataObject):
 187     def __init__(self, record):
 188         super().__init__("", record)
 189
 190     def _parse(self, record):
 191         return super()._parse(record)
 192
 193     def expand(self, param={}):
 194         obj = super().expand(param)
 195         param.update(**obj)
 196         return {}
 197
 198 class Condition(DataObject):
 199     def __init__(self, record):
 200         super().__init__("", record)
 201
 202     def _parse(self, record):
 203         super()._parse(record)
 204         if "range" not in self.ctrl_field:
 205             raise Exception("condition body must contains valid range case")
 206         if "true" not in self.ctrl_field:
 207             raise Exception("condition body must contains 'True' handling case")
 208
 209
 210     def expand(self, param={}):
 211         self.__range_lst = self.ctrl_field["range"].get(param)
 212         if param["index"] in self.__range_lst:
 213             return self.ctrl_field["true"].expand(param)
 214         elif "else" in self.ctrl_field:
 215             return self.ctrl_field["else"].expand(param)
 216         return {}
 217
 218 class ArrayObject(DataObject):
 219     def __init__(self, record,
 220                  nested_array = False,
 221                  el_factory = lambda x: DataObject.create("", x)):
 222         self._el_factory = el_factory
 223         self._nested_array = nested_array
 224
 225         super().__init__("", record)
 226
 227     def _parse(self, record):
 228         if not isinstance(record, list):
 229             raise Exception(f"{type(self).__name__} require array input")
 230
 231         self.content = []
 232         for x in record:
 233             self.content.append(self._el_factory(x))
 234
 235     def expand(self, param={}):
 236         result = []
 237         for x in self.content:
 238             obj = x.expand(param)
 239             if isinstance(obj, list) and not self._nested_array:
 240                 result += [*obj]
 241             else:
 242                 result.append(obj)
 243
 244         return result
 245
 246 class MemoryMapObject(DataObject):
 247     class GranuleObject(DataObject):
 248         def __init__(self, record):
 249             super().__init__("", record)
 250
 251         def _parse(self, record):
 252             self.__granules = {}
 253             for k, v in record.items():
 254                 self.__granules[k] = DataObject.create(k, v)
 255
 256         def expand(self, param={}):
 257             granules = {}
 258             for k, v in self.__granules.items():
 259                 val = v.expand(param)
 260
 261                 if not isinstance(val, int):
 262                     raise Exception("The granule definition must be either integer or int-castable string")
 263
 264                 granules[k] = val
 265
 266             return {**granules}
 267
 268     def __init__(self, record):
 269         super().__init__("", record)
 270
 271     def _parse(self, record):
 272         for k, v in record.items():
 273             if k.startswith("$"):
 274                 self.ctrl_field[k.strip("$")] = FieldType.create(k, v)
 275             elif k.startswith("@"):
 276                 self.ctrl_field[k.strip("@")] = DataObject.create(k, v)
 277
 278         if "granule" in record:
 279             self.__g = MemoryMapObject.GranuleObject(record["granule"])
 280
 281         if "regions" in record:
 282             self.__regions = ArrayObject(record["regions"])
 283
 284         if "width" in record:
 285             self.__width = DataObject.create("width", record["width"])
 286
 287     def __process(self, start_addr, idx, regions, size_lookahead = False):
 288         if idx >= len(regions):
 289             raise Exception("Unbounded region definition")
 290
 291         e = regions[idx]
 292
 293         if "block" in e:
 294             b = e["block"] - 1
 295             start_addr = (start_addr + b) & ~b
 296
 297         if "start" not in e:
 298             e["start"] = start_addr
 299         elif e["start"] < start_addr:
 300             raise Exception(f"starting addr {hex(e['start'])} overrlapping with {hex(start_addr)}")
 301         else:
 302             start_addr = e["start"]
 303
 304         if "size" not in e:
 305             if size_lookahead:
 306                 raise Exception("could not infer size from unbounded region")
 307             tmp_addr = self.__process(start_addr, idx + 1, regions, size_lookahead=True)
 308             e["size"] = tmp_addr - start_addr
 309
 310         if not size_lookahead:
 311             start_addr += e["size"]
 312
 313         return start_addr
 314
 315     def expand(self, param={}):
 316         super().expand(param)
 317
 318         g = self.__g.expand(param)
 319
 320         param["granule"] = g
 321
 322         width = self.__width.expand(param)
 323         if not isinstance(width, int):
 324             raise Exception("'width' attribute must be integer")
 325
 326         regions = self.__regions.expand(param)
 327
 328         start_addr = 0
 329         for i in range(len(regions)):
 330             start_addr = self.__process(start_addr, i, regions)
 331
 332         if math.log2(start_addr) > width:
 333             raise Exception("memory size larger than speicified address width")
 334
 335         return {
 336             "granule": g,
 337             "regions": regions
 338         }
 339
 340 class ForEachIndexObject(DataObject):
 341     def __init__(self, name, record):
 342         super().__init__(name, record)
 343         self.steps = []
 344         for k, v in record.items():
 345             self.steps.append(DataObject.create(k, v))
 346
 347     def _parse(self, record):
 348         super()._parse(record)
 349
 350     def expand(self, param={}):
 351         if "index" not in param:
 352             raise Exception(f"'{type(self).__name__}' require parameter 'index'.")
 353         obj = {}
 354         for cond in self.steps:
 355             obj.update(**cond.expand(param))
 356         return obj
 357
 358 class RawObject(DataObject):
 359     def __init__(self, name, record):
 360         super().__init__(name, record)
 361
 362     def _parse(self, record):
 363         return super()._parse(record)
 364
 365     def expand(self, param={}):
 366         return super().expand(param)
 367
 368 class PlainOldObject(DataObject):
 369     def __init__(self, name, record):
 370         super().__init__(name, record)
 371
 372     def _parse(self, record):
 373         return super()._parse(record)
 374
 375     def expand(self, param={}):
 376         return super().expand(param)
 377
 378 class RangedObject(DataObject):
 379     def __init__(self, name, record):
 380         super().__init__(name, record)
 381
 382     def _parse(self, record):
 383         super()._parse(record)
 384
 385     def expand(self, param={}):
 386         if "range" not in self.ctrl_field:
 387             raise Exception("RangedObject with ranged type must have 'range' field defined")
 388
 389         out_lst = []
 390         indices = self.ctrl_field["range"].get(param)
 391         for i in indices:
 392             param["index"] = i
 393             out_lst.append(super().expand(param))
 394
 395         return out_lst
 396
 397 def aligned(v, a):
 398     return v & ~(a - 1)
 399
 400 class TemplateExpander:
 401     def __init__(self, template_path, project_path, arch) -> None:
 402         self.arch = arch
 403         self.tbase_path = template_path.joinpath(arch)
 404         self.pbase_path = project_path
 405
 406         self.__helper_fns = {
 407             "align": aligned,
 408             "hex": lambda x: hex(x)
 409         }
 410
 411         self.__load_config()
 412         self.__load_mappings()
 413
 414     def __load_config(self):
 415         self.data = {}
 416         cfg_file: Path = self.tbase_path.joinpath("config.json")
 417         if not cfg_file.is_file():
 418             raise Exception(f"config not found. ({cfg_file})")
 419
 420         obj = json.loads(cfg_file.read_text())
 421         for k, v in obj.items():
 422             o = DataObject.create(k, v).expand()
 423             self.data[k] = o
 424
 425     def __load_mappings(self):
 426         self.mapping = {}
 427         mapping_file: Path = self.tbase_path.joinpath("mappings")
 428         if not mapping_file.is_file():
 429             raise Exception(f"config not found. ({mapping_file})")
 430
 431         with mapping_file.open() as f:
 432             for l in f:
 433                 l = l.strip()
 434
 435                 if not l:
 436                     continue
 437
 438                 src, dest = l.split("->")
 439                 src = src.strip()
 440
 441                 if src in self.mapping:
 442                     raise Exception(f"repeating entry ({src})")
 443
 444                 self.mapping[src] = dest.strip()
 445
 446     def render(self):
 447         for k, v in self.mapping.items():
 448             src: Path = self.tbase_path.joinpath(k)
 449             dest: Path = self.pbase_path.joinpath(v)
 450             if not src.is_file():
 451                 continue
 452
 453             if not dest.parent.exists():
 454                 dest.parent.mkdir(parents=True)
 455
 456             self.data["template"] = k
 457             template = jinja2.Template(src.read_text(), trim_blocks=True)
 458             template.globals.update(**self.__helper_fns)
 459             out = template.render(data=self.data)
 460
 461             dest.write_text(out)
 462
 463             print(f"rendered: {k} -> {v}")
 464
 465 import pprint
 466
 467 def main():
 468     parser = argparse.ArgumentParser()
 469     parser.add_argument("--arch", default='i386')
 470     parser.add_argument("-twd", "--template_dir", default=str(Path.cwd()))
 471     parser.add_argument("-pwd", "--project_dir", default=str(Path.cwd()))
 472
 473     args = parser.parse_args()
 474
 475     expander = TemplateExpander(Path(args.template_dir), Path(args.project_dir), args.arch)
 476
 477     expander.render()
 478     # pprint.pprint(expander.data)
 479
 480 if __name__ == "__main__":
 481     main()