lunaix-os/scripts/expand.py

   1 import jinja2
   2 import re
   3 import argparse
   4 import math
   5 import json
   6 from pathlib import Path
   7 from abc import ABC, abstractmethod
   8
   9 class Preprocessor:
  10     reHex = re.compile(r"^0x([0-9a-fA-F]+)$")
  11     reGranuel = re.compile(r"^(?P<num>[0-9]+)@(?P<g>.+)$")
  12     reMacroRef = re.compile(r"^\*(?P<var>[a-zA-z0-9]+)$")
  13     reInt = re.compile(r"^[0-9]+$")
  14     def __init__(self) -> None:
  15         pass
  16
  17     @staticmethod
  18     def expand_str(s: str, param_dict):
  19         if Preprocessor.reInt.match(s) is not None:
  20             return int(s)
  21
  22         mo = Preprocessor.reHex.match(s)
  23         if mo is not None:
  24             return int(s, 16)
  25
  26         mo = Preprocessor.reGranuel.match(s)
  27         if mo is not None:
  28             mg = mo.groupdict()
  29             num = int(mg['num'])
  30             granuel = param_dict["granule"][mg['g']]
  31             return num * granuel
  32
  33         mo = Preprocessor.reMacroRef.match(s)
  34         if mo is not None:
  35             mg = mo.groupdict()
  36             return param_dict[mg['var']]
  37
  38         return s.format_map(param_dict)
  39
  40 class DataObject(ABC):
  41     def __init__(self, name, record):
  42         self.key = name
  43         self._record = record
  44         self.user_field = {}
  45         self.ctrl_field = {}
  46         self._parse(record)
  47
  48     @staticmethod
  49     def create(record):
  50         return DataObject.create("", record)
  51
  52     @staticmethod
  53     def create(key, record):
  54         if PrimitiveType.can_create(record):
  55             return PrimitiveType(record)
  56
  57         name = key
  58         t = name if "$type" not in record else record['$type']
  59
  60         if "$name" in record:
  61             name = record["$name"].strip()
  62
  63         if not key.startswith('@') and "$type" not in record:
  64             return PlainOldObject(name, record)
  65
  66         t = t.strip("@")
  67         if t == "list":
  68             return RangedObject(name, record)
  69         elif t == "foreach":
  70             return ForEachIndexObject(name, record)
  71         elif t == "case_range_index":
  72             return Condition(record)
  73         elif t == "data":
  74             return PlainOldObject(name, record)
  75         elif t == "define":
  76             return VariableDeclarationObject(record)
  77         elif t == "memory_map":
  78             return MemoryMapObject(record)
  79         else:
  80             return RawObject(name, record)
  81
  82     def _parse(self, record):
  83         for k, v in record.items():
  84             if k.startswith("$"):
  85                 self.ctrl_field[k.strip("$")] = FieldType.create(k, v)
  86             elif k.startswith("@"):
  87                 self.ctrl_field[k.strip("@")] = DataObject.create(k, v)
  88             else:
  89                 self.user_field[k] = DataObject.create(k, v)
  90
  91     def expand(self, param={}):
  92         obj2 = {}
  93         for f in self.ctrl_field.values():
  94             if not isinstance(f, DataObject):
  95                 continue
  96             obj2.update(**f.expand(param))
  97
  98         obj = {}
  99         _param = {**param, **obj2}
 100         for k, v in self.user_field.items():
 101             if isinstance(v, DataObject):
 102                 obj[k] = v.expand(_param)
 103             else:
 104                 obj[k] = v
 105
 106         return {**obj, **obj2}
 107
 108
 109 class FieldType:
 110     def __init__(self, record) -> None:
 111         self._record = record
 112         self._parse(record)
 113
 114     @staticmethod
 115     def create(field, value):
 116         if field == "$range":
 117             return RangeType(value)
 118         else:
 119             return value
 120
 121     @abstractmethod
 122     def _parse(self, record):
 123         pass
 124
 125     @abstractmethod
 126     def get(self, param):
 127         pass
 128
 129     def getraw(self):
 130         return self.__record
 131
 132 class PrimitiveType(DataObject):
 133     def __init__(self, record) -> None:
 134         super().__init__("", record)
 135
 136     @staticmethod
 137     def can_create(value):
 138         return type(value) in (str, int, bool)
 139
 140     def _parse(self, record):
 141         if type(record) not in (str, int, bool):
 142             raise Exception(f"{type(self).__name__} require primitive type input")
 143         self.val = record
 144
 145         if type(record) == str:
 146             self.__get_fn = self.__process_str
 147         else:
 148             self.__get_fn = lambda x: self.val
 149
 150     def __process_str(self, param):
 151         return Preprocessor.expand_str(self.val, param)
 152
 153     def expand(self, param={}):
 154         return self.__get_fn(param)
 155
 156 class RangeType(FieldType):
 157     def __init__(self, record) -> None:
 158         self.__ranged_component = re.compile(r"^(?P<index>[^.]+)$|^(?P<start>.+?)\.\.(?P<end>.+)$")
 159         super().__init__(record)
 160
 161     def _parse(self, record):
 162         return super()._parse(record)
 163
 164     def get(self, param):
 165         record = self._record.strip('[]')
 166
 167         self.__value=[]
 168         for component in record.split(','):
 169             component = component.strip()
 170             mo = self.__ranged_component.match(component)
 171             if mo is None:
 172                 raise Exception(f"value '{component}' is not valid range component")
 173
 174             mo = mo.groupdict()
 175             if mo["index"] is not None:
 176                 self.__value.append(Preprocessor.expand_str(mo['index'], param))
 177             else:
 178                 start = Preprocessor.expand_str(mo['start'], param)
 179                 end = Preprocessor.expand_str(mo['end'], param)
 180                 self.__value += [x for x in range(start, end + 1)]
 181         return self.__value
 182
 183     def getraw(self):
 184         return self._record
 185
 186 class VariableDeclarationObject(DataObject):
 187     def __init__(self, record):
 188         super().__init__("", record)
 189
 190     def _parse(self, record):
 191         return super()._parse(record)
 192
 193     def expand(self, param={}):
 194         obj = super().expand(param)
 195         param.update(**obj)
 196         return {}
 197
 198 class Condition(DataObject):
 199     def __init__(self, record):
 200         super().__init__("", record)
 201
 202     def _parse(self, record):
 203         super()._parse(record)
 204         if "range" not in self.ctrl_field:
 205             raise Exception("condition body must contains valid range case")
 206         if "true" not in self.ctrl_field:
 207             raise Exception("condition body must contains 'True' handling case")
 208
 209
 210     def expand(self, param={}):
 211         self.__range_lst = self.ctrl_field["range"].get(param)
 212         if param["index"] in self.__range_lst:
 213             return self.ctrl_field["true"].expand(param)
 214         elif "else" in self.ctrl_field:
 215             return self.ctrl_field["else"].expand(param)
 216         return {}
 217
 218 class ArrayObject(DataObject):
 219     def __init__(self, record,
 220                  nested_array = False,
 221                  el_factory = lambda x: DataObject.create("", x)):
 222         self._el_factory = el_factory
 223         self._nested_array = nested_array
 224
 225         super().__init__("", record)
 226
 227     def _parse(self, record):
 228         if not isinstance(record, list):
 229             raise Exception(f"{type(self).__name__} require array input")
 230
 231         self.content = []
 232         for x in record:
 233             self.content.append(self._el_factory(x))
 234
 235     def expand(self, param={}):
 236         result = []
 237         for x in self.content:
 238             obj = x.expand(param)
 239             if isinstance(obj, list) and not self._nested_array:
 240                 result += [*obj]
 241             else:
 242                 result.append(obj)
 243
 244         return result
 245
 246 class MemoryMapObject(DataObject):
 247     class GranuleObject(DataObject):
 248         def __init__(self, record):
 249             super().__init__("", record)
 250
 251         def _parse(self, record):
 252             self.__granules = {}
 253             for k, v in record.items():
 254                 self.__granules[k] = DataObject.create(k, v)
 255
 256         def expand(self, param={}):
 257             granules = {}
 258             for k, v in self.__granules.items():
 259                 val = v.expand(param)
 260
 261                 if not isinstance(val, int):
 262                     raise Exception("The granule definition must be either integer or int-castable string")
 263
 264                 granules[k] = val
 265
 266             return {**granules}
 267
 268     def __init__(self, record):
 269         super().__init__("", record)
 270
 271     def _parse(self, record):
 272         if "granule" in self._record:
 273             self.__g = MemoryMapObject.GranuleObject(self._record["granule"])
 274         if "regions" in self._record:
 275             self.__regions = ArrayObject(self._record["regions"])
 276         if "width" in self._record:
 277             self.__width = DataObject.create("width", self._record["width"])
 278
 279     def __process(self, start_addr, idx, regions, size_lookahead = False):
 280         if idx >= len(regions):
 281             raise Exception("Unbounded region definition")
 282
 283         e = regions[idx]
 284
 285         if "boundary" in e:
 286             b = e["boundary"] - 1
 287             start_addr = (start_addr + b) & ~b
 288
 289         if "start" not in e:
 290             e["start"] = start_addr
 291         elif e["start"] < start_addr:
 292             raise Exception(f"starting addr {hex(e['start'])} overrlapping with {hex(start_addr)}")
 293         else:
 294             start_addr = e["start"]
 295
 296         if "size" not in e:
 297             if size_lookahead:
 298                 raise Exception("could not infer size from unbounded region")
 299             tmp_addr = self.__process(start_addr, idx + 1, regions, size_lookahead=True)
 300             e["size"] = tmp_addr - start_addr
 301
 302         if not size_lookahead:
 303             start_addr += e["size"]
 304
 305         return start_addr
 306
 307     def expand(self, param={}):
 308         g = self.__g.expand(param)
 309
 310         param["granule"] = g
 311
 312         width = self.__width.expand(param)
 313         if not isinstance(width, int):
 314             raise Exception("'width' attribute must be integer")
 315
 316         regions = self.__regions.expand(param)
 317
 318         start_addr = 0
 319         for i in range(len(regions)):
 320             start_addr = self.__process(start_addr, i, regions)
 321
 322         if math.log2(start_addr) > width:
 323             raise Exception("memory size larger than speicified address width")
 324
 325         return {
 326             "granule": g,
 327             "regions": regions
 328         }
 329
 330 class ForEachIndexObject(DataObject):
 331     def __init__(self, name, record):
 332         super().__init__(name, record)
 333         self.steps = []
 334         for k, v in record.items():
 335             self.steps.append(DataObject.create(k, v))
 336
 337     def _parse(self, record):
 338         super()._parse(record)
 339
 340     def expand(self, param={}):
 341         if "index" not in param:
 342             raise Exception(f"'{type(self).__name__}' require parameter 'index'.")
 343         obj = {}
 344         for cond in self.steps:
 345             obj.update(**cond.expand(param))
 346         return obj
 347
 348 class RawObject(DataObject):
 349     def __init__(self, name, record):
 350         super().__init__(name, record)
 351
 352     def _parse(self, record):
 353         return super()._parse(record)
 354
 355     def expand(self, param={}):
 356         return super().expand(param)
 357
 358 class PlainOldObject(DataObject):
 359     def __init__(self, name, record):
 360         super().__init__(name, record)
 361
 362     def _parse(self, record):
 363         return super()._parse(record)
 364
 365     def expand(self, param={}):
 366         return super().expand(param)
 367
 368 class RangedObject(DataObject):
 369     def __init__(self, name, record):
 370         super().__init__(name, record)
 371
 372     def _parse(self, record):
 373         super()._parse(record)
 374
 375     def expand(self, param={}):
 376         if "range" not in self.ctrl_field:
 377             raise Exception("RangedObject with ranged type must have 'range' field defined")
 378
 379         out_lst = []
 380         indices = self.ctrl_field["range"].get(param)
 381         for i in indices:
 382             param["index"] = i
 383             out_lst.append(super().expand(param))
 384
 385         return out_lst
 386
 387 def aligned(v, a):
 388     return v & ~(a - 1)
 389
 390 class TemplateExpander:
 391     def __init__(self, template_path, project_path, arch) -> None:
 392         self.arch = arch
 393         self.tbase_path = template_path.joinpath(arch)
 394         self.pbase_path = project_path
 395
 396         self.__helper_fns = {
 397             "align": aligned,
 398             "hex": lambda x: hex(x)
 399         }
 400
 401         self.__load_config()
 402         self.__load_mappings()
 403
 404     def __load_config(self):
 405         self.data = {}
 406         cfg_file: Path = self.tbase_path.joinpath("config.json")
 407         if not cfg_file.is_file():
 408             raise Exception(f"config not found. ({cfg_file})")
 409
 410         obj = json.loads(cfg_file.read_text())
 411         for k, v in obj.items():
 412             o = DataObject.create(k, v).expand()
 413             self.data[k] = o
 414
 415     def __load_mappings(self):
 416         self.mapping = {}
 417         mapping_file: Path = self.tbase_path.joinpath("mappings")
 418         if not mapping_file.is_file():
 419             raise Exception(f"config not found. ({mapping_file})")
 420
 421         with mapping_file.open() as f:
 422             for l in f:
 423                 l = l.strip()
 424
 425                 if not l:
 426                     continue
 427
 428                 src, dest = l.split("->")
 429                 src = src.strip()
 430
 431                 if src in self.mapping:
 432                     raise Exception(f"repeating entry ({src})")
 433
 434                 self.mapping[src] = dest.strip()
 435
 436     def render(self):
 437         for k, v in self.mapping.items():
 438             src: Path = self.tbase_path.joinpath(k)
 439             dest: Path = self.pbase_path.joinpath(v)
 440             if not src.is_file():
 441                 continue
 442
 443             if not dest.parent.exists():
 444                 dest.parent.mkdir(parents=True)
 445
 446             self.data["template"] = k
 447             template = jinja2.Template(src.read_text(), trim_blocks=True)
 448             template.globals.update(**self.__helper_fns)
 449             out = template.render(data=self.data)
 450
 451             dest.write_text(out)
 452
 453             print(f"rendered: {k} -> {v}")
 454
 455 import pprint
 456
 457 def main():
 458     parser = argparse.ArgumentParser()
 459     parser.add_argument("--arch", default='i386')
 460     parser.add_argument("-twd", "--template_dir", default=str(Path.cwd()))
 461     parser.add_argument("-pwd", "--project_dir", default=str(Path.cwd()))
 462
 463     args = parser.parse_args()
 464
 465     expander = TemplateExpander(Path(args.template_dir), Path(args.project_dir), args.arch)
 466
 467     expander.render()
 468     # pprint.pprint(expander.data)
 469
 470 if __name__ == "__main__":
 471     main()