X-Git-Url: https://scm.lunaixsky.com/lunaix-os.git/blobdiff_plain/28c176b668c841a3b7fb093faccf0efa39257603..669e8fc63dd1aa7fe4c830b1d40371a1ab47fc4f:/lunaix-os/kernel/mm/procvm.c

diff --git a/lunaix-os/kernel/mm/procvm.c b/lunaix-os/kernel/mm/procvm.c
index 5b7afad..8081971 100644
--- a/lunaix-os/kernel/mm/procvm.c
+++ b/lunaix-os/kernel/mm/procvm.c
@@ -4,11 +4,25 @@
 #include <lunaix/mm/page.h>
 #include <lunaix/mm/mmap.h>
 #include <lunaix/process.h>
+#include <lunaix/syslog.h>
 
-#include <sys/mm/mm_defs.h>
+#include <asm/mm_defs.h>
 
 #include <klibc/string.h>
 
+#define alloc_pagetable_trace(ptep, pte, ord, level)                        \
+    ({                                                                      \
+        alloc_kpage_at(ptep, pte, ord);                                     \
+    })
+
+#define free_pagetable_trace(ptep, pte, level)                              \
+    ({                                                                      \
+        struct leaflet* leaflet = pte_leaflet_aligned(pte);                 \
+        assert(leaflet_order(leaflet) == 0);                                \
+        leaflet_return(leaflet);                                            \
+        set_pte(ptep, null_pte);                                            \
+    })
+
 struct proc_mm*
 procvm_create(struct proc_info* proc) {
     struct proc_mm* mm = vzalloc(sizeof(struct proc_mm));
@@ -29,14 +43,162 @@ __ptep_advancement(struct leaflet* leaflet, int level)
     return (1 << (leaflet_order(leaflet) % shifts)) - 1;
 }
 
-static ptr_t
-vmscpy(ptr_t dest_mnt, ptr_t src_mnt, bool only_kernel)
+static inline int
+__descend(ptr_t dest_mnt, ptr_t src_mnt, ptr_t va, bool alloc)
+{
+    pte_t *dest, *src, pte;
+
+    int i = 0;
+    while (!pt_last_level(i))
+    {
+        dest = mklntep_va(i, dest_mnt, va);
+        src  = mklntep_va(i, src_mnt, va);
+        pte  = pte_at(src);
+
+        if (!pte_isloaded(pte) || pte_huge(pte)) {
+            break;
+        }
+
+        if (alloc && pte_isnull(pte_at(dest))) {
+            alloc_pagetable_trace(dest, pte, 0, i);
+        }
+
+        i++;
+    }
+
+    return i;
+}
+
+static inline void
+copy_leaf(pte_t* dest, pte_t* src, pte_t pte, int level)
+{
+    struct leaflet* leaflet;
+
+    set_pte(dest, pte);
+
+    if (!pte_isloaded(pte)) {
+        return;
+    }
+
+    leaflet = pte_leaflet(pte);
+    assert(leaflet_refcount(leaflet));
+    
+    if (leaflet_ppfn(leaflet) == pte_ppfn(pte)) {
+        leaflet_borrow(leaflet);
+    }
+}
+
+static inline void
+copy_root(pte_t* dest, pte_t* src, pte_t pte, int level)
+{
+    alloc_pagetable_trace(dest, pte, 0, level);
+}
+
+static void
+vmrcpy(ptr_t dest_mnt, ptr_t src_mnt, struct mm_region* region)
+{
+    pte_t *src, *dest;
+    ptr_t loc;
+    int level;
+    struct leaflet* leaflet;
+
+    loc  = region->start;
+    src  = mkptep_va(src_mnt, loc);
+    dest = mkptep_va(dest_mnt, loc);
+
+    level = __descend(dest_mnt, src_mnt, loc, true);
+
+    while (loc < region->end)
+    {
+        pte_t pte = *src;
+
+        if (pte_isnull(pte)) {
+            goto cont;
+        } 
+        
+        if (pt_last_level(level) || pte_huge(pte)) {
+            copy_leaf(dest, src, pte, level);
+            goto cont;
+        }
+        
+        if (!pt_last_level(level)) {
+            copy_root(dest, src, pte, level);
+
+            src = ptep_step_into(src);
+            dest = ptep_step_into(dest);
+            level++;
+
+            continue;
+        }
+        
+    cont:
+        loc += lnt_page_size(level);
+        while (ptep_vfn(src) == MAX_PTEN - 1) {
+            assert(level > 0);
+            src = ptep_step_out(src);
+            dest = ptep_step_out(dest);
+            level--;
+        }
+
+        src++;
+        dest++;
+    }
+}
+
+static void
+vmrfree(ptr_t vm_mnt, struct mm_region* region)
 {
-    pte_t* ptep_dest    = mkl0tep(mkptep_va(dest_mnt, 0));
-    pte_t* ptep         = mkl0tep(mkptep_va(src_mnt, 0));
-    pte_t* ptepd_kernel = mkl0tep(mkptep_va(dest_mnt, KERNEL_RESIDENT));
-    pte_t* ptep_kernel  = mkl0tep(mkptep_va(src_mnt, KERNEL_RESIDENT));
+    pte_t *src, *end;
+    ptr_t loc;
+    int level;
+    struct leaflet* leaflet;
+
+    loc  = region->start;
+    src  = mkptep_va(vm_mnt, region->start);
+    end  = mkptep_va(vm_mnt, region->end);
+
+    level = __descend(vm_mnt, vm_mnt, loc, false);
+
+    while (src < end)
+    {
+        pte_t pte = *src;
+        ptr_t pa  = pte_paddr(pte);
+
+        if (pte_isnull(pte)) {
+            goto cont;
+        } 
+
+        if (!pt_last_level(level) && !pte_huge(pte)) {
+            src = ptep_step_into(src);
+            level++;
+
+            continue;
+        }
+
+        set_pte(src, null_pte);
+        
+        if (pte_isloaded(pte)) {
+            leaflet = pte_leaflet_aligned(pte);
+            leaflet_return(leaflet);
+
+            src += __ptep_advancement(leaflet, level);
+        }
+
+    cont:
+        while (ptep_vfn(src) == MAX_PTEN - 1) {
+            src = ptep_step_out(src);
+            free_pagetable_trace(src, pte_at(src), level);
+            
+            level--;
+        }
+
+        src++;
+    }
+}
 
+static void
+vmscpy(struct proc_mm* dest_mm, struct proc_mm* src_mm)
+{
     // Build the self-reference on dest vms
 
     /* 
@@ -73,150 +235,93 @@ vmscpy(ptr_t dest_mnt, ptr_t src_mnt, bool only_kernel)
      *      Note: PML4: 2 extra steps
      *            PML5: 3 extra steps
     */
+
+    ptr_t  dest_mnt, src_mnt;
+    
+    dest_mnt = dest_mm->vm_mnt;
+    assert(dest_mnt);
+
     pte_t* ptep_ssm     = mkl0tep_va(VMS_SELF, dest_mnt);
-    pte_t* ptep_sms     = mkl1tep_va(VMS_SELF, dest_mnt) + VMS_SELF_L0TI;
-    pte_t  pte_sms      = mkpte_prot(KERNEL_DATA);
+    pte_t* ptep_smx     = mkl1tep_va(VMS_SELF, dest_mnt);
+    pte_t  pte_sms      = mkpte_prot(KERNEL_PGTAB);
 
-    pte_sms = alloc_kpage_at(ptep_ssm, pte_sms, 0);
-    set_pte(ptep_sms, pte_sms);    
+    pte_sms = alloc_pagetable_trace(ptep_ssm, pte_sms, 0, 0);
+    set_pte(&ptep_smx[VMS_SELF_L0TI], pte_sms);
     
     tlb_flush_kernel((ptr_t)dest_mnt);
-    tlb_flush_kernel((ptr_t)ptep_sms);
-
-    if (only_kernel) {
-        ptep = ptep_kernel;
-        ptep_dest += ptep_vfn(ptep_kernel);
-    } else {
-        ptep++;
-        ptep_dest++;
+
+    if (!src_mm) {
+        goto done;
     }
 
-    int level = 0;
-    struct leaflet* leaflet;
+    src_mnt = src_mm->vm_mnt;
 
-    while (ptep < ptep_kernel)
+    struct mm_region *pos, *n;
+    llist_for_each(pos, n, &src_mm->regions, head)
     {
-        pte_t pte = *ptep;
-
-        if (pte_isnull(pte)) {
-            goto cont;
-        } 
-        
-        if (pt_last_level(level) || pte_huge(pte)) {
-            set_pte(ptep_dest, pte);
-
-            if (pte_isloaded(pte)) {
-                leaflet = pte_leaflet(pte);
-                assert(leaflet_refcount(leaflet));
-                
-                if (leaflet_ppfn(leaflet) == pte_ppfn(pte)) {
-                    leaflet_borrow(leaflet);
-                }
-            }
-        }
-        else if (!pt_last_level(level)) {
-            alloc_kpage_at(ptep_dest, pte, 0);
+        vmrcpy(dest_mnt, src_mnt, pos);
+    }
 
-            ptep = ptep_step_into(ptep);
-            ptep_dest = ptep_step_into(ptep_dest);
-            level++;
+done:;
+    procvm_link_kernel(dest_mnt);
+    
+    dest_mm->vmroot = pte_paddr(pte_sms);
+}
 
-            continue;
-        }
-        
-    cont:
-        while (ptep_vfn(ptep) == MAX_PTEN - 1) {
-            assert(level > 0);
-            ptep = ptep_step_out(ptep);
-            ptep_dest = ptep_step_out(ptep_dest);
-            level--;
-        }
+static void
+__purge_vms_residual(struct proc_mm* mm, int level, ptr_t va)
+{
+    pte_t *ptep, pte;
+    ptr_t _va;
 
-        ptep++;
-        ptep_dest++;
+    if (level >= MAX_LEVEL) {
+        return;
     }
 
-    // Ensure we step back to L0T
-    assert(!level);
-    assert(ptep_dest == ptepd_kernel);
-    
-    // Carry over the kernel (exclude last two entry)
-    unsigned int i = ptep_vfn(ptep);
-    while (i++ < MAX_PTEN) {
-        pte_t pte = *ptep;
+    ptep = mklntep_va(level, mm->vm_mnt, va);
 
-        if (l0tep_impile_vmnts(ptep)) {
-            goto _cont;
+    for (unsigned i = 0; i < LEVEL_SIZE; i++, ptep++) 
+    {
+        pte = pte_at(ptep);
+        if (pte_isnull(pte) || !pte_isloaded(pte)) {
+            continue;
         }
 
-        assert(!pte_isnull(pte));
-
-        // Ensure it is a next level pagetable,
-        //  we MAY relax this later allow kernel
-        //  to have huge leaflet mapped at L0T
-        leaflet = pte_leaflet_aligned(pte);
-        assert(leaflet_order(leaflet) == 0);
+        if (lntep_implie_vmnts(ptep, lnt_page_size(level))) {
+            continue;
+        }
 
-        set_pte(ptep_dest, pte);
-        leaflet_borrow(leaflet);
-    
-    _cont:
-        ptep++;
-        ptep_dest++;
+        _va = va + (i * lnt_page_size(level));
+        __purge_vms_residual(mm, level + 1, _va);
+        
+        set_pte(ptep, null_pte);
+        leaflet_return(pte_leaflet_aligned(pte));
     }
-
-    return pte_paddr(pte_sms);
 }
 
 static void
-vmsfree(ptr_t vm_mnt)
+vmsfree(struct proc_mm* mm)
 {
     struct leaflet* leaflet;
-    pte_t* ptep_head    = mkl0tep(mkptep_va(vm_mnt, 0));
-    pte_t* ptep_self    = mkl0tep(mkptep_va(vm_mnt, VMS_SELF));
-    pte_t* ptep_kernel  = mkl0tep(mkptep_va(vm_mnt, KERNEL_RESIDENT));
+    struct mm_region *pos, *n;
+    ptr_t vm_mnt;
+    pte_t* ptep_self;
+    
+    vm_mnt    = mm->vm_mnt;
+    ptep_self = mkl0tep_va(vm_mnt, VMS_SELF);
 
-    int level = 0;
-    pte_t* ptep = ptep_head;
-    while (ptep < ptep_kernel)
+    // first pass: free region mappings
+    llist_for_each(pos, n, &mm->regions, head)
     {
-        pte_t pte = *ptep;
-        ptr_t pa  = pte_paddr(pte);
-
-        if (pte_isnull(pte)) {
-            goto cont;
-        } 
-
-        if (!pt_last_level(level) && !pte_huge(pte)) {
-            ptep = ptep_step_into(ptep);
-            level++;
-
-            continue;
-        }
-
-        if (pte_isloaded(pte)) {
-            leaflet = pte_leaflet_aligned(pte);
-            leaflet_return(leaflet);
-
-            ptep += __ptep_advancement(leaflet, level);
-        }
+        vmrfree(vm_mnt, pos);
+    }
 
-    cont:
-        while (ptep_vfn(ptep) == MAX_PTEN - 1) {
-            ptep = ptep_step_out(ptep);
-            leaflet = pte_leaflet_aligned(pte_at(ptep));
-            
-            assert(leaflet_order(leaflet) == 0);
-            leaflet_return(leaflet);
-            
-            level--;
-        }
+    procvm_unlink_kernel(vm_mnt);
 
-        ptep++;
-    }
+    // free up all allocated tables on intermediate levels
+    __purge_vms_residual(mm, 0, 0);
 
-    leaflet = pte_leaflet_aligned(pte_at(ptep_self));
-    leaflet_return(leaflet);
+    free_pagetable_trace(ptep_self, pte_at(ptep_self), 0);
 }
 
 static inline void
@@ -239,6 +344,11 @@ __detach_from_current_vms(struct proc_mm* guest_mm)
     }
 }
 
+void
+procvm_prune_vmr(ptr_t vm_mnt, struct mm_region* region)
+{
+    vmrfree(vm_mnt, region);
+}
 
 void
 procvm_dupvms_mount(struct proc_mm* mm) {
@@ -251,14 +361,20 @@ procvm_dupvms_mount(struct proc_mm* mm) {
    
     mm->heap = mm_current->heap;
     mm->vm_mnt = VMS_MOUNT_1;
-    mm->vmroot = vmscpy(VMS_MOUNT_1, VMS_SELF, false);
     
+    vmscpy(mm, mm_current);  
     region_copy_mm(mm_current, mm);
 }
 
 void
 procvm_mount(struct proc_mm* mm)
 {
+    // if current mm is already active
+    if (active_vms(mm->vm_mnt)) {
+        return;
+    }
+    
+    // we are double mounting
     assert(!mm->vm_mnt);
     assert(mm->vmroot);
 
@@ -272,9 +388,13 @@ procvm_mount(struct proc_mm* mm)
 void
 procvm_unmount(struct proc_mm* mm)
 {
+    if (active_vms(mm->vm_mnt)) {
+        return;
+    }
+    
     assert(mm->vm_mnt);
-
     vms_unmount(VMS_MOUNT_1);
+    
     struct proc_mm* mm_current = vmspace(__current);
     if (mm_current) {
         mm_current->guest_mm = NULL;
@@ -291,22 +411,28 @@ procvm_initvms_mount(struct proc_mm* mm)
     __attach_to_current_vms(mm);
 
     mm->vm_mnt = VMS_MOUNT_1;
-    mm->vmroot = vmscpy(VMS_MOUNT_1, VMS_SELF, true);
+    vmscpy(mm, NULL);
 }
 
 void
 procvm_unmount_release(struct proc_mm* mm) {
     ptr_t vm_mnt = mm->vm_mnt;
     struct mm_region *pos, *n;
+
     llist_for_each(pos, n, &mm->regions, head)
     {
         mem_sync_pages(vm_mnt, pos, pos->start, pos->end - pos->start, 0);
+    }
+
+    vmsfree(mm);
+
+    llist_for_each(pos, n, &mm->regions, head)
+    {
         region_release(pos);
     }
 
-    vfree(mm);
-    vmsfree(vm_mnt);
     vms_unmount(vm_mnt);
+    vfree(mm);
 
     __detach_from_current_vms(mm);
 }
@@ -315,7 +441,6 @@ void
 procvm_mount_self(struct proc_mm* mm) 
 {
     assert(!mm->vm_mnt);
-    assert(!mm->guest_mm);
 
     mm->vm_mnt = VMS_SELF;
 }
@@ -350,11 +475,13 @@ procvm_enter_remote(struct remote_vmctx* rvmctx, struct proc_mm* mm,
 
     pte_t* rptep = mkptep_va(vm_mnt, remote_base);
     pte_t* lptep = mkptep_va(VMS_SELF, rvmctx->local_mnt);
-    unsigned int pattr = region_pteprot(region);
+
+    pte_t pte, rpte = null_pte;
+    rpte = region_tweakpte(region, rpte);
 
     for (size_t i = 0; i < size_pn; i++)
     {
-        pte_t pte = vmm_tryptep(rptep, PAGE_SIZE);
+        pte = vmm_tryptep(rptep, PAGE_SIZE);
         if (pte_isloaded(pte)) {
             set_pte(lptep, pte);
             continue;
@@ -362,7 +489,7 @@ procvm_enter_remote(struct remote_vmctx* rvmctx, struct proc_mm* mm,
 
         ptr_t pa = ppage_addr(pmm_alloc_normal(0));
         set_pte(lptep, mkpte(pa, KERNEL_DATA));
-        set_pte(rptep, mkpte(pa, pattr));
+        set_pte(rptep, pte_setpaddr(rpte, pa));
     }
 
     return vm_mnt;