Lots of improvements to 64-bit Sortix.

Fixed 64-bit-ness bug in BSR() and BSF().
Added 64-bit system call stubs in libmaxsi.
Added a Elf64 program loader.
Fixed uninitialized memory bug in the scheduler.
x64/boot.s now takes care of user-space memory permissions.
Fixed bug in x64/syscall.s

That's right. The system now boots in 64-bit mode.

It is horribly unstable, though.

libmaxsi/error.cpp

@@ -60,7 +60,7 @@ namespace Maxsi
 				case EISDIR: return (char*) "Is a directory";
 				case EPERM: return (char*) "Operation not permitted";
 				case EIO: return (char*) "Input/output error";
-				case ENOEXEC: return (char*) "Not executable";
+				case ENOEXEC: return (char*) "Exec format error";
 				case EACCESS: return (char*) "Permission denied";
 				case ESRCH: return (char*) "No such process";
 				case ENOTTY: return (char*) "Not a tty";

libmaxsi/heap.cpp

@@ -30,7 +30,7 @@
 #define HEAP_GROWS_DOWNWARDS
 #endif
 
-#define PARANOIA 0
+#define PARANOIA 1
 
 #ifdef SORTIX_KERNEL
 #include <sortix/platform.h>
@@ -141,7 +141,7 @@ namespace Maxsi
 			ASSERT(Value > 0);
 			for ( size_t I = 8*sizeof(size_t); I > 0; I-- )
 			{
-				if ( Value & ( 1 << (I-1) ) ) { return I-1; }
+				if ( Value & ( 1UL << (I-1) ) ) { return I-1; }
 			}
 			return 0;
 #else
@@ -158,7 +158,7 @@ namespace Maxsi
 			ASSERT(Value > 0);
 			for ( size_t I = 0; I < 8*sizeof(size_t); I++ )
 			{
-				if ( Value & ( 1 << I ) ) { return I; }
+				if ( Value & ( 1UL << I ) ) { return I; }
 			}
 			return 0;
 #else
@@ -268,6 +268,7 @@ namespace Maxsi
 
 		bool Chunk::IsSane()
 		{
+			if ( !size ) { return false; }
 			size_t binindex = BSR(size);
 			Trailer* trailer = GetTrailer();
 			if ( trailer->size != size ) { return false; }
@@ -314,6 +315,9 @@ namespace Maxsi
 
 		bool ValidateHeap()
 		{
+			bool foundbin[NUMBINS];
+			for ( size_t i = 0; i < NUMBINS; i++ ) { foundbin[i] = false; }
+
 			#ifdef HEAP_GROWS_DOWNWARDS
 			Chunk* chunk = (Chunk*) (wilderness + wildernesssize);
 			while ( (addr_t) chunk < heapstart )
@@ -322,10 +326,28 @@ namespace Maxsi
 			while ( (addr_t) chunk < wilderness - wildernesssize )
 			#endif
 			{
+				size_t timesfound = 0;
+				for ( size_t i = 0; i < NUMBINS; i++ )
+				{
+					if ( chunk == bins[i] ) { foundbin[i] = true; timesfound++; }
+				}
+				if ( 1 < timesfound ) { return false; }
+
 				if ( !chunk->IsSane() ) { return false; }
 				chunk = chunk->RightNeighbor();
 			}
 
+			for ( size_t i = 0; i < NUMBINS; i++ )
+			{
+				if ( !bins[i] )
+				{
+					if ( foundbin[i] ) { return false; }
+					continue;
+				}
+				if ( !foundbin[i] ) { return false; }
+				if ( !bins[i]->IsSane() ) { return false; }
+			}
+
 			return true;
 		}
 

libmaxsi/hsrc/syscall.h

@@ -137,67 +137,91 @@ namespace Maxsi
 	#define DEFN_SYSCALL0(type, fn, num) \
 	inline type fn() \
 	{ \
-		return 0; \
+		type a; \
+		asm volatile("int $0x80" : "=a" (a) : "0" (num)); \
+		return a; \
 	}
 
 	#define DEFN_SYSCALL1(type, fn, num, P1) \
 	inline type fn(P1 p1) \
 	{ \
-		return 0; \
+		type a; \
+		asm volatile("int $0x80" : "=a" (a) : "0" (num)); \
+		return a; \
 	}
 
 	#define DEFN_SYSCALL2(type, fn, num, P1, P2) \
 	inline type fn(P1 p1, P2 p2) \
 	{ \
-		return 0; \
+		type a; \
+		asm volatile("int $0x80" : "=a" (a) : "0" (num)); \
+		return a; \
 	}
 
 	#define DEFN_SYSCALL3(type, fn, num, P1, P2, P3) \
 	inline type fn(P1 p1, P2 p2, P3 p3) \
 	{ \
-		return 0; \
+		type a; \
+		asm volatile("int $0x80" : "=a" (a) : "0" (num)); \
+		return a; \
 	}
 
 	#define DEFN_SYSCALL4(type, fn, num, P1, P2, P3, P4) \
 	inline type fn(P1 p1, P2 p2, P3 p3, P4 p4) \
 	{ \
-		return 0; \
+		type a; \
+		asm volatile("int $0x80" : "=a" (a) : "0" (num)); \
+		return a; \
 	}
 
 	#define DEFN_SYSCALL5(type, fn, num, P1, P2, P3, P4, P5) \
 	inline type fn(P1 p1, P2 p2, P3 p3, P4 p4, P5 p5) \
 	{ \
-		return 0; \
+		type a; \
+		asm volatile("int $0x80" : "=a" (a) : "0" (num)); \
+		return a; \
 	}
 
 	#define DEFN_SYSCALL0_VOID(fn, num) \
 	inline void fn() \
 	{ \
+		size_t a; \
+		asm volatile("int $0x80" : "=a" (a) : "0" (num)); \
 	}
 
 	#define DEFN_SYSCALL1_VOID(fn, num, P1) \
 	inline void fn(P1 p1) \
 	{ \
+		size_t a; \
+		asm volatile("int $0x80" : "=a" (a) : "0" (num)); \
 	}
 
 	#define DEFN_SYSCALL2_VOID(fn, num, P1, P2) \
 	inline void fn(P1 p1, P2 p2) \
 	{ \
+		size_t a; \
+		asm volatile("int $0x80" : "=a" (a) : "0" (num)); \
 	}
 
 	#define DEFN_SYSCALL3_VOID(fn, num, P1, P2, P3) \
 	inline void fn(P1 p1, P2 p2, P3 p3) \
 	{ \
+		size_t a; \
+		asm volatile("int $0x80" : "=a" (a) : "0" (num)); \
 	}
 
 	#define DEFN_SYSCALL4_VOID(fn, num, P1, P2, P3, P4) \
 	inline void fn(P1 p1, P2 p2, P3 p3, P4 p4) \
 	{ \
+		size_t a; \
+		asm volatile("int $0x80" : "=a" (a) : "0" (num)); \
 	}
 
 	#define DEFN_SYSCALL5_VOID(fn, num, P1, P2, P3, P4, P5) \
 	inline void fn(P1 p1, P2 p2, P3 p3, P4 p4, P5 p5) \
 	{ \
+		size_t a; \
+		asm volatile("int $0x80" : "=a" (a) : "0" (num)); \
 	}
 
 #endif

sortix/elf.cpp

@@ -111,9 +111,84 @@ namespace Sortix
 			return entry;
 		}
 
-		addr_t Construct64(Process* /*process*/, const void* /*file*/, size_t /*filelen*/)
+		addr_t Construct64(Process* process, const void* file, size_t filelen)
 		{
+			#ifndef PLATFORM_X64
+			Error::Set(ENOEXEC);
 			return 0;
+			#else
+			if ( filelen < sizeof(Header64) ) { return 0; }
+			const Header64* header = (const Header64*) file;
+
+			// Check for little endian.
+			if ( header->dataencoding != DATA2LSB ) { return 0; }
+			if ( header->version != CURRENTVERSION ) { return 0; }
+
+			addr_t entry = header->entry;
+
+			// Find the location of the program headers.
+			addr_t phtbloffset = header->programheaderoffset;
+			if ( filelen < phtbloffset ) { return 0; }
+			addr_t phtblpos = ((addr_t) file) + phtbloffset;
+			size_t phsize = header->programheaderentrysize;
+			const ProgramHeader64* phtbl = (const ProgramHeader64*) phtblpos;
+
+			// Validate that all program headers are present.
+			uint16_t numprogheaders = header->numprogramheaderentries;
+			size_t neededfilelen = phtbloffset + numprogheaders * phsize;
+			if ( filelen < neededfilelen ) { return 0; }
+
+			// Prepare the process for execution (clean up address space, etc.)
+			process->ResetForExecute();
+
+			// Flush the TLB such that no stale information from the last
+			// address space is used when creating the new one.
+			Memory::Flush();
+
+			// Create all the segments in the final process.
+			// TODO: Handle errors on bad/malicious input or out-of-mem!
+			for ( uint16_t i = 0; i < numprogheaders; i++ )
+			{
+				const ProgramHeader64* pht = &(phtbl[i]);
+				if ( pht->type != PT_LOAD ) { continue; }
+				addr_t virtualaddr = pht->virtualaddr;
+				addr_t mapto = Page::AlignDown(virtualaddr);
+				addr_t mapbytes = virtualaddr - mapto + pht->memorysize;
+				ASSERT(pht->offset % pht->align == virtualaddr % pht->align);
+				ASSERT(pht->offset + pht->filesize < filelen);
+				ASSERT(pht->filesize <= pht->memorysize);
+
+				ProcessSegment* segment = new ProcessSegment;
+				if ( segment == NULL ) { return 0; }
+				segment->position = mapto;
+				segment->size = Page::AlignUp(mapbytes);
+
+				if ( segment->Intersects(process->segments) )
+				{
+					delete segment;
+					return 0;
+				}
+
+				if ( !Memory::MapRangeUser(mapto, mapbytes))
+				{
+					return 0;
+				}
+
+				// Insert our newly allocated memory into the processes segment
+				// list such that it can be reclaimed later.
+				if ( process->segments ) { process->segments->prev = segment; }
+				segment->next = process->segments;
+				process->segments = segment;
+
+				// Copy as much data as possible and memset the rest to 0.
+				byte* memdest = (byte*) virtualaddr;
+				byte* memsource = (byte*) ( ((addr_t)file) + pht->offset);
+				Maxsi::Memory::Copy(memdest, memsource, pht->filesize);
+				Maxsi::Memory::Set(memdest + pht->filesize, 0, pht->memorysize - pht->filesize);
+			}
+
+			return entry;
+			#endif
 		}
 
 		addr_t Construct(Process* process, const void* file, size_t filelen)

sortix/elf.h

@@ -37,7 +37,9 @@ namespace Sortix
 			unsigned char fileclass;
 			unsigned char dataencoding;
 			unsigned char version;
-			unsigned char padding[9];
+			unsigned char osabi;
+			unsigned char abiversion;
+			unsigned char padding[7];
 		};
 
 		const unsigned char CLASSNONE = 0;
@@ -64,6 +66,23 @@ namespace Sortix
 			uint16_t sectionheaderstringindex;
 		};
 
+		struct Header64 : public Header
+		{
+			uint16_t type;
+			uint16_t machine;
+			uint32_t version;
+			uint64_t entry;
+			uint64_t programheaderoffset;
+			uint64_t sectionheaderoffset;
+			uint32_t flags;
+			uint16_t elfheadersize;
+			uint16_t programheaderentrysize;
+			uint16_t numprogramheaderentries;
+			uint16_t sectionheaderentrysize;
+			uint16_t numsectionheaderentries;
+			uint16_t sectionheaderstringindex;
+		};
+
 		struct SectionHeader32
 		{
 			uint32_t name;
@@ -78,6 +97,20 @@ namespace Sortix
 			uint32_t entsize;
 		};
 
+		struct SectionHeader64
+		{
+			uint32_t name;
+			uint32_t type;
+			uint64_t flags;
+			uint64_t addr;
+			uint64_t offset;
+			uint64_t size;
+			uint32_t link;
+			uint32_t info;
+			uint64_t addralign;
+			uint64_t entsize;
+		};
+
 		const uint32_t SHT_NULL = 0;
 		const uint32_t SHT_PROGBITS = 1;
 		const uint32_t SHT_SYMTAB = 2;
@@ -107,6 +140,18 @@ namespace Sortix
 			uint32_t align;
 		};
 
+		struct ProgramHeader64
+		{
+			uint32_t type;
+			uint32_t flags;
+			uint64_t offset;
+			uint64_t virtualaddr;
+			uint64_t physicaladdr;
+			uint64_t filesize;
+			uint64_t memorysize;
+			uint64_t align;
+		};
+
 		const uint32_t PT_NULL = 0;
 		const uint32_t PT_LOAD = 1;
 		const uint32_t PT_DYNAMIC = 2;

sortix/kernel.cpp

@@ -262,13 +262,6 @@ namespace Sortix
 		// Set up the initial ram disk.
 		InitRD::Init(initrd, initrdsize);
 
-#ifdef PLATFORM_X64
-		Log::Print("Halt: There is no program loader for 64-bit Sortix\n");
-		Log::Print("Sorry, it simply isn't possible to fully boot Sortix in x64 mode yet.\n");
-		Log::Print("x64 may be working when Sortix 0.5 comes out, or try the git master.\n");
-		while(true);
-#endif
-
 		// Alright, now the system's drivers are loaded and initialized. It is
 		// time to load the initial user-space programs and start execution of
 		// the actual operating system.
@@ -278,11 +271,11 @@ namespace Sortix
 
 		// Create an address space for the idle process.
 		addr_t idleaddrspace = Memory::Fork();
-		if ( !idleaddrspace ) { Panic("could not fork an idle process"); }
+		if ( !idleaddrspace ) { Panic("could not fork an idle process address space"); }
 
 		// Create an address space for the initial process.
 		addr_t initaddrspace = Memory::Fork();
-		if ( !initaddrspace ) { Panic("could not fork an initial process"); }
+		if ( !initaddrspace ) { Panic("could not fork an initial process address space"); }
 
 		// Create the system idle process.
 		Process* idle = new Process;

sortix/scheduler.cpp

@@ -69,6 +69,7 @@ namespace Sortix
 			// simpler code.
 			currentthread = dummythread;
 			firstrunnablethread = NULL;
+			firstsleepingthread = NULL;
 			idlethread = NULL;
 			hacksigintpending = false;
 

sortix/x64/boot.s

@@ -66,17 +66,19 @@ multiboot_entry:
 	movl %cr3, %edi
 
 	# Set the initial page tables.
+	# Note that we OR with 0x7 here to allow user-space access, except in the
+	# first 2 MiB. We also do this with 0x200 to allow forking the page.
 
 	# Page-Map Level 4
-	movl $0x2003, (%edi)
+	movl $0x2207, (%edi)
 	addl $0x1000, %edi
 
 	# Page-Directory Pointer Table
-	movl $0x3003, (%edi)
+	movl $0x3207, (%edi)
 	addl $0x1000, %edi
 
 	# Page-Directory
-	movl $0x4003, (%edi)
+	movl $0x4207, (%edi)
 	addl $0x1000, %edi
 
 	# Page-Table

sortix/x64/syscall.s

@@ -83,7 +83,7 @@ syscall_handler:
 valid_rax:
 	# Read a system call function pointer.
 	xorq %rbp, %rbp
-	movq syscall_list(%rbp,%rax,4), %rax
+	movq syscall_list(%rbp,%rax,8), %rax
 
 	# Oh how nice, user-space put the parameters in: rdi, rsi, rdx, rcx, r8, r9