nortti
/
sortix-mirror
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement threading primitives that truly sleep. 

The idle thread is now actually run when the system is idle because it
truly goes idle. The idle thread is made power efficient by using the hlt
instruction rather than a busy loop.

The new futex(2) system call is used to implement fast user-space mutexes,
condition variables, and semaphores. The same backend and design is used as
kutexes for truly sleeping kernel mutexes and condition variables.

The new exit_thread(2) flag EXIT_THREAD_FUTEX_WAKE wakes a futex.

Sleeping on clocks in the kernel now uses timers for true sleep.

The interrupt worker thread now truly sleeps when idle.

Kernel threads are now named.

This is a compatible ABI change.
This commit is contained in:
Jonas 'Sortie' Termansen 2021-02-06 21:23:43 +01:00
parent 29b14378e8
commit 5e7605fad2
49 changed files with 967 additions and 558 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -174,7 +174,7 @@ sysroot-system: sysroot-fsh sysroot-base-headers
echo 'ID=sortix' && \
echo 'VERSION_ID="$(VERSION)"' && \
echo 'PRETTY_NAME="Sortix $(VERSION)"' && \
echo 'SORTIX_ABI=1.0' && \
echo 'SORTIX_ABI=1.1' && \
true) > "$(SYSROOT)/etc/sortix-release"
echo /etc/sortix-release >> "$(SYSROOT)/tix/manifest/system"
ln -sf sortix-release "$(SYSROOT)/etc/os-release"

1
kernel/Makefile
View File

@ -82,7 +82,6 @@ alarm.o \
clock.o \
com.o \
copy.o \
$(CPUDIR)/kthread.o \
descriptor.o \
disk/ahci/ahci.o \
disk/ahci/hba.o \

96
kernel/clock.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2016, 2017, 2018 Jonas 'Sortie' Termansen.
* Copyright (c) 2013, 2016, 2017, 2018, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -25,6 +25,7 @@
#include <sortix/kernel/kernel.h>
#include <sortix/kernel/kthread.h>
#include <sortix/kernel/signal.h>
#include <sortix/kernel/thread.h>
#include <sortix/kernel/timer.h>
#include <sortix/kernel/worker.h>
@ -260,60 +261,61 @@ bool Clock::TryCancel(Timer* timer)
return active;
}
static void timer_wakeup(Clock* /*clock*/, Timer* /*timer*/, void* ctx)
{
Thread* thread = (Thread*) ctx;
thread->timer_woken = true;
kthread_wake_futex(thread);
}
// TODO: We need some method for threads to sleep for real but still be
// interrupted by signals.
struct timespec Clock::SleepDelay(struct timespec duration)
{
struct timespec start_advancement;
struct timespec elapsed = timespec_nul();
bool start_advancement_set = false;
while ( timespec_lt(elapsed, duration) )
{
if ( start_advancement_set )
{
if ( Signal::IsPending() )
return duration;
kthread_yield();
}
LockClock();
if ( !start_advancement_set )
{
start_advancement = current_advancement;
start_advancement_set = true;
}
elapsed = timespec_sub(current_advancement, start_advancement);
UnlockClock();
}
LockClock();
struct timespec start_advancement = current_advancement;
UnlockClock();
Thread* thread = CurrentThread();
thread->futex_woken = false;
thread->timer_woken = false;
Timer timer;
timer.Attach(this);
struct itimerspec timerspec;
timerspec.it_value = duration;
timerspec.it_interval.tv_sec = 0;
timerspec.it_interval.tv_nsec = 0;
int timer_flags = TIMER_FUNC_INTERRUPT_HANDLER;
timer.Set(&timerspec, NULL, timer_flags, timer_wakeup, thread);
kthread_wait_futex_signal();
timer.Cancel();
LockClock();
struct timespec end_advancement = current_advancement;
UnlockClock();
struct timespec elapsed = timespec_sub(end_advancement, start_advancement);
if ( timespec_lt(elapsed, duration) )
return timespec_sub(duration, elapsed);
return timespec_nul();
}
// TODO: We need some method for threads to sleep for real but still be
// interrupted by signals.
struct timespec Clock::SleepUntil(struct timespec expiration)
{
while ( true )
{
LockClock();
struct timespec now = current_time;
UnlockClock();
if ( timespec_le(expiration, now) )
break;
if ( Signal::IsPending() )
return timespec_sub(expiration, now);
kthread_yield();
}
Thread* thread = CurrentThread();
thread->futex_woken = false;
thread->timer_woken = false;
Timer timer;
timer.Attach(this);
struct itimerspec timerspec;
timerspec.it_value = expiration;
timerspec.it_interval.tv_sec = 0;
timerspec.it_interval.tv_nsec = 0;
int timer_flags = TIMER_ABSOLUTE | TIMER_FUNC_INTERRUPT_HANDLER;
timer.Set(&timerspec, NULL, timer_flags, timer_wakeup, thread);
kthread_wait_futex_signal();
timer.Cancel();
LockClock();
struct timespec now = current_time;
UnlockClock();
struct timespec remaining = timespec_sub(expiration, now);
if ( timespec_lt(timespec_nul(), remaining) )
return remaining;
return timespec_nul();
}

20
kernel/copy.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2012, 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -129,6 +129,24 @@ bool CopyFromUser(void* kdst_ptr, const void* usersrc_ptr, size_t count)
return result;
}
bool ReadAtomicFromUser(int* kdst_ptr, const int* usersrc_ptr)
{
uintptr_t usersrc = (uintptr_t) usersrc_ptr;
if ( usersrc & (sizeof(int) - 1) )
return errno = EINVAL, false;
Process* process = CurrentProcess();
assert(IsInProcessAddressSpace(process));
ScopedLock lock(&process->segment_lock);
struct segment* segment = FindSegment(process, usersrc);
if ( !segment || !(segment->prot & PROT_READ) )
return errno = EFAULT, false;
size_t segment_available = segment->addr + segment->size - usersrc;
if ( segment_available < sizeof(int) )
return errno = EFAULT, false;
*kdst_ptr = __atomic_load_n(usersrc_ptr, __ATOMIC_SEQ_CST);
return true;
}
bool CopyToKernel(void* kdst, const void* ksrc, size_t count)
{
memcpy(kdst, ksrc, count);

3
kernel/include/sortix/exit.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -51,6 +51,7 @@ struct exit_thread
#define EXIT_THREAD_TLS_UNMAP (1<<3)
#define EXIT_THREAD_PROCESS (1<<4)
#define EXIT_THREAD_DUMP_CORE (1<<5)
#define EXIT_THREAD_FUTEX_WAKE (1<<6)
#ifdef __cplusplus
} /* extern "C" */

35
kernel/include/sortix/futex.h Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright (c) 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*
* sortix/futex.h
* Fast userspace mutexes.
*/
#ifndef INCLUDE_SORTIX_FUTEX_H
#define INCLUDE_SORTIX_FUTEX_H
#include <sys/cdefs.h>
#define FUTEX_WAIT 1
#define FUTEX_WAKE 2
#define FUTEX_ABSOLUTE (1 << 8)
#define FUTEX_CLOCK(clock) (clock << 24)
#define FUTEX_GET_OP(op) ((op) & 0xFF)
#define FUTEX_GET_CLOCK(op) ((op) >> 24)
#endif

3
kernel/include/sortix/kernel/copy.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2012, 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -28,6 +28,7 @@ bool CopyToUser(void* userdst, const void* ksrc, size_t count);
bool CopyFromUser(void* kdst, const void* usersrc, size_t count);
bool CopyToKernel(void* kdst, const void* ksrc, size_t count);
bool CopyFromKernel(void* kdst, const void* ksrc, size_t count);
bool ReadAtomicFromUser(int* kdst, const int* usersrc);
bool ZeroKernel(void* kdst, size_t count);
bool ZeroUser(void* userdst, size_t count);
char* GetStringFromUser(const char* str);

6
kernel/include/sortix/kernel/interrupt.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, 2012, 2013, 2014, 2017 Jonas 'Sortie' Termansen.
* Copyright (c) 2011, 2012, 2013, 2014, 2017, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -110,12 +110,12 @@ inline bool IsCPUInterrupted()
inline bool SetEnabled(bool is_enabled)
{
bool wasenabled = IsEnabled();
bool was_enabled = IsEnabled();
if ( is_enabled )
Enable();
else
Disable();
return wasenabled;
return was_enabled;
}
void RegisterHandler(unsigned int index, struct interrupt_handler* handler);

19
kernel/include/sortix/kernel/kthread.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2012, 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -26,15 +26,18 @@
namespace Sortix {
extern "C" {
class Thread;
inline static void kthread_yield(void) { asm volatile ("int $129"); }
void kthread_yield();
void kthread_wait_futex();
void kthread_wait_futex_signal();
void kthread_wake_futex(Thread* thread);
__attribute__((noreturn)) void kthread_exit();
typedef unsigned kthread_mutex_t;
typedef int kthread_mutex_t;
const kthread_mutex_t KTHREAD_MUTEX_INITIALIZER = 0;
unsigned kthread_mutex_trylock(kthread_mutex_t* mutex);
bool kthread_mutex_trylock(kthread_mutex_t* mutex);
void kthread_mutex_lock(kthread_mutex_t* mutex);
unsigned long kthread_mutex_lock_signal(kthread_mutex_t* mutex);
bool kthread_mutex_lock_signal(kthread_mutex_t* mutex);
void kthread_mutex_unlock(kthread_mutex_t* mutex);
struct kthread_cond_elem;
typedef struct kthread_cond_elem kthread_cond_elem_t;
@ -46,12 +49,10 @@ struct kthread_cond
typedef struct kthread_cond kthread_cond_t;
const kthread_cond_t KTHREAD_COND_INITIALIZER = { NULL, NULL };
void kthread_cond_wait(kthread_cond_t* cond, kthread_mutex_t* mutex);
unsigned long kthread_cond_wait_signal(kthread_cond_t* cond, kthread_mutex_t* mutex);
bool kthread_cond_wait_signal(kthread_cond_t* cond, kthread_mutex_t* mutex);
void kthread_cond_signal(kthread_cond_t* cond);
void kthread_cond_broadcast(kthread_cond_t* cond);
} /* extern "C" */
class ScopedLock
{
public:

7
kernel/include/sortix/kernel/process.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, 2012, 2013, 2014, 205, 2016 Jonas 'Sortie' Termansen.
* Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -138,6 +138,11 @@ public:
size_t threads_not_exiting_count;
bool threads_exiting;
public:
kthread_mutex_t futex_lock;
Thread* futex_first_waiting;
Thread* futex_last_waiting;
public:
struct segment* segments;
size_t segments_used;

20
kernel/include/sortix/kernel/scheduler.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, 2012, 2013, 2014, 2017 Jonas 'Sortie' Termansen.
* Copyright (c) 2011, 2012, 2013, 2014, 2017, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -29,28 +29,16 @@ class Thread;
} // namespace Sortix
namespace Sortix {
enum ThreadState { NONE, RUNNABLE, BLOCKING, DEAD };
enum ThreadState { NONE, RUNNABLE, FUTEX_WAITING, DEAD };
} // namespace Sortix
namespace Sortix {
namespace Scheduler {
#if defined(__i386__) || defined(__x86_64__)
static inline void Yield()
{
asm volatile ("int $129");
}
__attribute__ ((noreturn))
static inline void ExitThread()
{
asm volatile ("int $132");
__builtin_unreachable();
}
#endif
void Switch(struct interrupt_context* intctx);
void SwitchTo(struct interrupt_context* intctx, Thread* new_thread);
void SetThreadState(Thread* thread, ThreadState state);
void SetThreadState(Thread* thread, ThreadState state, bool wake_only = false);
void SetSignalPending(Thread* thread, unsigned long is_pending);
ThreadState GetThreadState(Thread* thread);
void SetIdleThread(Thread* thread);
void SetInitProcess(Process* init);

3
kernel/include/sortix/kernel/syscall.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen.
* Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -89,6 +89,7 @@ int sys_fstatvfs(int, struct statvfs*);
int sys_fstatvfsat(int, const char*, struct statvfs*, int);
int sys_fsync(int);
int sys_ftruncate(int, off_t);
int sys_futex(int*, int, int, const struct timespec*);
int sys_futimens(int, const struct timespec*);
int sys_getdnsconfig(struct dnsconfig*);
gid_t sys_getegid(void);

28
kernel/include/sortix/kernel/thread.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011-2016, 2018 Jonas 'Sortie' Termansen.
* Copyright (c) 2011-2016, 2018, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -39,19 +39,30 @@ class Process;
class Thread;
// These functions create a new kernel process but doesn't start it.
Thread* CreateKernelThread(Process* process, struct thread_registers* regs);
Thread* CreateKernelThread(Process* process, struct thread_registers* regs,
const char* name);
Thread* CreateKernelThread(Process* process, void (*entry)(void*), void* user,
const char* name, size_t stacksize = 0);
Thread* CreateKernelThread(void (*entry)(void*), void* user, const char* name,
size_t stacksize = 0);
Thread* CreateKernelThread(void (*entry)(void*), void* user, size_t stacksize = 0);
// This function can be used to start a thread from the above functions.
void StartKernelThread(Thread* thread);
// Alternatively, these functions both create and start the thread.
Thread* RunKernelThread(Process* process, struct thread_registers* regs);
Thread* RunKernelThread(Process* process, struct thread_registers* regs,
const char* name);
Thread* RunKernelThread(Process* process, void (*entry)(void*), void* user,
const char* name, size_t stacksize = 0);
Thread* RunKernelThread(void (*entry)(void*), void* user, const char* name,
size_t stacksize = 0);
Thread* RunKernelThread(void (*entry)(void*), void* user, size_t stacksize = 0);
enum yield_operation
{
YIELD_OPERATION_NONE,
YIELD_OPERATION_WAIT_FUTEX,
YIELD_OPERATION_WAIT_FUTEX_SIGNAL,
};
class Thread
{
@ -60,6 +71,7 @@ public:
~Thread();
public:
const char* name;
uintptr_t system_tid;
uintptr_t yield_to_tid;
struct thread_registers registers;
@ -87,6 +99,12 @@ public:
bool has_saved_signal_mask;
Clock execute_clock;
Clock system_clock;
uintptr_t futex_address;
bool futex_woken;
bool timer_woken;
Thread* futex_prev_waiting;
Thread* futex_next_waiting;
enum yield_operation yield_operation;
public:
void HandleSignal(struct interrupt_context* intctx);

5
kernel/include/sortix/syscall.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen.
* Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -188,6 +188,7 @@
#define SYSCALL_SOCKET 165
#define SYSCALL_GETDNSCONFIG 166
#define SYSCALL_SETDNSCONFIG 167
#define SYSCALL_MAX_NUM 168 /* index of highest constant + 1 */
#define SYSCALL_FUTEX 168
#define SYSCALL_MAX_NUM 169 /* index of highest constant + 1 */
#endif

2
kernel/interlock.cpp
View File

@ -23,12 +23,14 @@
namespace Sortix {
// TODO: This is likely not the most optimal way to perform these operations.
// TODO: Just discard this whole thing in favor of __atomic_?
ilret_t InterlockedModify(unsigned long* ptr,
ilockfunc f,
unsigned long user)
{
unsigned long old_value, new_value;
// TODO: Migrate to __atomic.
do
{
old_value = *((volatile unsigned long*) ptr); /* TODO: Need volatile? */

17
kernel/interrupt.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, 2012, 2013, 2014, 2016, 2017 Jonas 'Sortie' Termansen.
* Copyright (c) 2011-2017, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -24,6 +24,7 @@
#include <sortix/kernel/interrupt.h>
#include <sortix/kernel/kernel.h>
#include <sortix/kernel/kthread.h>
#include <sortix/kernel/thread.h>
namespace Sortix {
namespace Interrupt {
@ -33,12 +34,16 @@ bool interrupt_worker_thread_boost = false;
static struct interrupt_work* first;
static struct interrupt_work* last;
static bool interrupt_worker_idle = false;
void WorkerThread(void* /*user*/)
{
Thread* thread = CurrentThread();
assert(Interrupt::IsEnabled());
while ( true )
{
thread->futex_woken = false;
thread->timer_woken = false;
struct interrupt_work* work;
Interrupt::Disable();
work = first;
@ -47,8 +52,9 @@ void WorkerThread(void* /*user*/)
Interrupt::Enable();
if ( !work )
{
// TODO: Make this thread not run until work arrives.
kthread_yield();
interrupt_worker_idle = true;
kthread_wait_futex();
interrupt_worker_idle = false;
continue;
}
while ( work )
@ -67,6 +73,11 @@ void ScheduleWork(struct interrupt_work* work)
work->next = NULL;
last = work;
interrupt_worker_thread_boost = true;
if ( interrupt_worker_idle )
{
interrupt_worker_thread->futex_woken = true;
kthread_wake_futex(interrupt_worker_thread);
}
}
} // namespace Interrupt

21
kernel/kernel.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011-2017 Jonas 'Sortie' Termansen.
* Copyright (c) 2011-2018, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -193,7 +193,6 @@ extern "C" void KernelInit(unsigned long magic, multiboot_info_t* bootinfo_p)
// Display the logo.
Log::PrintF("\e[37;41m\e[2J");
Log::Center(BRAND_LOGO);
#if defined(__x86_64__)
// TODO: Remove this hack when qemu 1.4.x and 1.5.0 are obsolete.
// Verify that we are not running under a buggy qemu where the instruction
@ -375,6 +374,7 @@ extern "C" void KernelInit(unsigned long magic, multiboot_info_t* bootinfo_p)
// scheduler's set of runnable threads, but rather run whenever there is
// _nothing_ else to run on this CPU.
Thread* idlethread = AllocateThread();
idlethread->name = "idle";
idlethread->process = system;
idlethread->kernelstackpos = (addr_t) stack;
idlethread->kernelstacksize = STACK_SIZE;
@ -387,7 +387,7 @@ extern "C" void KernelInit(unsigned long magic, multiboot_info_t* bootinfo_p)
// Note that we don't do the work here: if all other threads are not running
// and this thread isn't runnable, then there is nothing to run. Therefore
// we must become the system idle thread.
RunKernelThread(BootThread, NULL);
RunKernelThread(BootThread, NULL, "boot");
// The time driver will run the scheduler on the next timer interrupt.
Time::Start();
@ -401,7 +401,14 @@ static void SystemIdleThread(void* /*user*/)
// Alright, we are now the system idle thread. If there is nothing to do,
// then we are run. Note that we must never do any real work here as the
// idle thread must always be runnable.
while(true);
while ( true )
{
#if defined(__i386__) || defined(__x86_64__)
asm volatile ("hlt");
#else
#warning "Implement a power efficient kernel idle thread"
#endif
}
}
static void BootThread(void* /*user*/)
@ -425,7 +432,7 @@ static void BootThread(void* /*user*/)
// Let's create the interrupt worker thread that executes additional work
// requested by interrupt handlers, where such work isn't safe.
Interrupt::interrupt_worker_thread =
RunKernelThread(Interrupt::WorkerThread, NULL);
RunKernelThread(Interrupt::WorkerThread, NULL, "interrupt");
if ( !Interrupt::interrupt_worker_thread )
Panic("Could not create interrupt worker");
@ -433,7 +440,7 @@ static void BootThread(void* /*user*/)
Worker::Init();
// Create a general purpose worker thread.
Thread* worker_thread = RunKernelThread(Worker::Thread, NULL);
Thread* worker_thread = RunKernelThread(Worker::Thread, NULL, "worker");
if ( !worker_thread )
Panic("Unable to create general purpose worker thread");
@ -644,7 +651,7 @@ static void BootThread(void* /*user*/)
init->addrspace = initaddrspace;
Scheduler::SetInitProcess(init);
Thread* initthread = RunKernelThread(init, InitThread, NULL);
Thread* initthread = RunKernelThread(init, InitThread, NULL, "main");
if ( !initthread )
Panic("Could not create init thread");

306
kernel/kthread.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2012, 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -17,6 +17,8 @@
* Utility and synchronization mechanisms for kernel threads.
*/
#include <limits.h>
#include <sortix/signal.h>
#include <sortix/kernel/kernel.h>
@ -27,8 +29,195 @@
#include <sortix/kernel/thread.h>
#include <sortix/kernel/worker.h>
#include "uart.h"
namespace Sortix {
void kthread_yield()
{
Thread* thread = CurrentThread();
thread->yield_operation = YIELD_OPERATION_NONE;
#if defined(__i386__) || defined(__x86_64__)
asm volatile ("int $129");
#else
#error "kthread_yield needs to be implemented"
#endif
}
void kthread_wait_futex()
{
Thread* thread = CurrentThread();
thread->yield_operation = YIELD_OPERATION_WAIT_FUTEX;
#if defined(__i386__) || defined(__x86_64__)
asm volatile ("int $129");
#else
#error "kthread_wait_futex needs to be implemented"
#endif
}
void kthread_wait_futex_signal()
{
Thread* thread = CurrentThread();
thread->yield_operation = YIELD_OPERATION_WAIT_FUTEX_SIGNAL;
#if defined(__i386__) || defined(__x86_64__)
asm volatile ("int $129");
#else
#error "kthread_wait_futex needs to be implemented"
#endif
}
void kthread_wake_futex(Thread* thread)
{
Scheduler::SetThreadState(thread, ThreadState::RUNNABLE, true);
}
static const int UNLOCKED = 0;
static const int LOCKED = 1;
static const int CONTENDED = 2;
void kthread_spinlock_lock(kthread_mutex_t* mutex)
{
while ( true )
{
int state = UNLOCKED;
int desired = LOCKED;
if ( __atomic_compare_exchange_n(mutex, &state, desired, false,
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) )
break;
}
}
void kthread_spinlock_unlock(kthread_mutex_t* mutex)
{
__atomic_store_n(mutex, UNLOCKED, __ATOMIC_SEQ_CST);
}
static bool kutex_wait(int* address, int value, bool signal)
{
// TODO: Use a per-mutex wait queue instead.
Thread* thread = CurrentThread();
Process* kernel_process = Scheduler::GetKernelProcess();
bool was_enabled = Interrupt::SetEnabled(false);
kthread_spinlock_lock(&kernel_process->futex_lock);
thread->futex_address = (uintptr_t) address;
thread->futex_woken = false;
thread->futex_prev_waiting = kernel_process->futex_last_waiting;
thread->futex_next_waiting = NULL;
(kernel_process->futex_last_waiting ?
kernel_process->futex_last_waiting->futex_next_waiting :
kernel_process->futex_first_waiting) = thread;
kernel_process->futex_last_waiting = thread;
kthread_spinlock_unlock(&kernel_process->futex_lock);
Interrupt::SetEnabled(was_enabled);
thread->timer_woken = false;
bool result = true;
if ( __atomic_load_n(address, __ATOMIC_SEQ_CST) == value )
{
if ( signal )
kthread_wait_futex();
else
kthread_wait_futex_signal();
}
Interrupt::SetEnabled(false);
kthread_spinlock_lock(&kernel_process->futex_lock);
if ( result && !thread->futex_woken )
{
if ( signal && Signal::IsPending() )
result = false;
}
thread->futex_address = 0;
thread->futex_woken = false;
(thread->futex_prev_waiting ?
thread->futex_prev_waiting->futex_next_waiting :
kernel_process->futex_first_waiting) = thread->futex_next_waiting;
(thread->futex_next_waiting ?
thread->futex_next_waiting->futex_prev_waiting :
kernel_process->futex_last_waiting) = thread->futex_prev_waiting;
thread->futex_prev_waiting = NULL;
thread->futex_next_waiting = NULL;
kthread_spinlock_unlock(&kernel_process->futex_lock);
Interrupt::SetEnabled(was_enabled);
return result;
}
static void kutex_wake(int* address, int count)
{
Process* kernel_process = Scheduler::GetKernelProcess();
bool was_enabled = Interrupt::SetEnabled(false);
kthread_spinlock_lock(&kernel_process->futex_lock);
for ( Thread* waiter = kernel_process->futex_first_waiting;
0 < count && waiter;
waiter = waiter->futex_next_waiting )
{
if ( waiter->futex_address == (uintptr_t) address )
{
waiter->futex_woken = true;
kthread_wake_futex(waiter);
if ( count != INT_MAX )
count--;
}
}
kthread_spinlock_unlock(&kernel_process->futex_lock);
Interrupt::SetEnabled(was_enabled);
}
bool kthread_mutex_trylock(kthread_mutex_t* mutex)
{
int state = UNLOCKED;
if ( !__atomic_compare_exchange_n(mutex, &state, LOCKED, false,
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) )
return false;
return true;
}
void kthread_mutex_lock(kthread_mutex_t* mutex)
{
int state = UNLOCKED;
int desired = LOCKED;
while ( !__atomic_compare_exchange_n(mutex, &state, desired, false,
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) )
{
if ( state == LOCKED &&
!__atomic_compare_exchange_n(mutex, &state, CONTENDED, false,
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) )
{
state = UNLOCKED;
continue;
}
desired = CONTENDED;
kutex_wait(mutex, CONTENDED, false);
state = UNLOCKED;
}
}
bool kthread_mutex_lock_signal(kthread_mutex_t* mutex)
{
int state = UNLOCKED;
int desired = LOCKED;
while ( !__atomic_compare_exchange_n(mutex, &state, desired, false,
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) )
{
if ( state == LOCKED &&
!__atomic_compare_exchange_n(mutex, &state, CONTENDED, false,
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) )
{
state = UNLOCKED;
continue;
}
desired = CONTENDED;
if ( !kutex_wait(mutex, CONTENDED, true) )
return false;
state = UNLOCKED;
}
return true;
}
void kthread_mutex_unlock(kthread_mutex_t* mutex)
{
if ( __atomic_exchange_n(mutex, UNLOCKED, __ATOMIC_SEQ_CST) == CONTENDED )
kutex_wake(mutex, 1);
}
// The kernel thread needs another stack to delete its own stack.
static void kthread_do_kill_thread(void* user)
{
@ -38,7 +227,7 @@ static void kthread_do_kill_thread(void* user)
FreeThread(thread);
}
extern "C" void kthread_exit()
void kthread_exit()
{
Process* process = CurrentProcess();
// Note: This requires all threads in this process to have been made by
@ -55,84 +244,109 @@ extern "C" void kthread_exit()
if ( is_last_to_exit )
process->OnLastThreadExit();
Worker::Schedule(kthread_do_kill_thread, CurrentThread());
Scheduler::ExitThread();
#if defined(__i386__) || defined(__x86_64__)
asm volatile ("int $132");
#else
#error "kthread_exit needs to be implemented"
#endif
__builtin_unreachable();
}
struct kthread_cond_elem
{
kthread_cond_elem_t* next;
volatile unsigned long woken;
kthread_cond_elem_t* prev;
int woken;
};
extern "C" void kthread_cond_wait(kthread_cond_t* cond, kthread_mutex_t* mutex)
void kthread_cond_wait(kthread_cond_t* cond, kthread_mutex_t* mutex)
{
kthread_cond_elem_t elem;
elem.next = NULL;
elem.prev = cond->last;
elem.woken = 0;
if ( cond->last ) { cond->last->next = &elem; }
if ( !cond->last ) { cond->first = &elem; }
if ( cond->last )
cond->last->next = &elem;
if ( !cond->first )
cond->first = &elem;
cond->last = &elem;
while ( !elem.woken )
kthread_mutex_unlock(mutex);
while ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) &&
kutex_wait(&elem.woken, 0, false) < 0 );
kthread_mutex_lock(mutex);
if ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) )
{
kthread_mutex_unlock(mutex);
Scheduler::Yield();
kthread_mutex_lock(mutex);
if ( elem.next )
elem.next->prev = elem.prev;
else
cond->last = elem.prev;
if ( elem.prev )
elem.prev->next = elem.next;
else
cond->first = elem.next;
}
}
extern "C" unsigned long kthread_cond_wait_signal(kthread_cond_t* cond,
kthread_mutex_t* mutex)
bool kthread_cond_wait_signal(kthread_cond_t* cond, kthread_mutex_t* mutex)
{
if ( Signal::IsPending() )
return 0;
return false;
kthread_cond_elem_t elem;
elem.next = NULL;
elem.prev = cond->last;
elem.woken = 0;
if ( cond->last ) { cond->last->next = &elem; }
if ( !cond->last ) { cond->first = &elem; }
if ( cond->last )
cond->last->next = &elem;
if ( !cond->first )
cond->first = &elem;
cond->last = &elem;
while ( !elem.woken )
kthread_mutex_unlock(mutex);
bool result = true;
while ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) &&
kutex_wait(&elem.woken, 0, false) < 0 )
{
if ( Signal::IsPending() )
{
if ( cond->first == &elem )
{
cond->first = elem.next;
if ( cond->last == &elem )
cond->last = NULL;
}
else
{
kthread_cond_elem_t* prev = cond->first;
while ( prev->next != &elem )
prev = prev->next;
prev->next = elem.next;
if ( cond->last == &elem )
cond->last = prev;
}
// Note that the thread still owns the mutex.
return 0;
result = false;
break;
}
kthread_mutex_unlock(mutex);
Scheduler::Yield();
kthread_mutex_lock(mutex);
}
return 1;
kthread_mutex_lock(mutex);
if ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) )
{
if ( elem.next )
elem.next->prev = elem.prev;
else
cond->last = elem.prev;
if ( elem.prev )
elem.prev->next = elem.next;
else
cond->first = elem.next;
}
return result;
}
extern "C" void kthread_cond_signal(kthread_cond_t* cond)
void kthread_cond_signal(kthread_cond_t* cond)
{
kthread_cond_elem_t* elem = cond->first;
if ( !elem ) { return; }
if ( !(cond->first = elem->next) ) { cond->last = NULL; }
elem->next = NULL;
elem->woken = 1;
if ( cond->first )
{
struct kthread_cond_elem* elem = cond->first;
if ( elem->next )
elem->next->prev = elem->prev;
else
cond->last = elem->prev;
cond->first = elem->next;
elem->next = NULL;
elem->prev = NULL;
__atomic_store_n(&elem->woken, 1, __ATOMIC_SEQ_CST);
kutex_wake(&elem->woken, 1);
}
}
extern "C" void kthread_cond_broadcast(kthread_cond_t* cond)
void kthread_cond_broadcast(kthread_cond_t* cond)
{
while ( cond->first ) { kthread_cond_signal(cond); }
while ( cond->first )
kthread_cond_signal(cond);
}
} // namespace Sortix

8
kernel/lfbtextbuffer.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen.
* Copyright (c) 2012, 2013, 2014, 2015, 2016, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -119,7 +119,8 @@ LFBTextBuffer* CreateLFBTextBuffer(uint8_t* lfb, uint32_t lfbformat,
return ret;
ret->queue_thread = true; // Visible to new thread.
if ( !RunKernelThread(kernel_process, LFBTextBuffer__RenderThread, ret) )
if ( !RunKernelThread(kernel_process, LFBTextBuffer__RenderThread, ret,
"console") )
ret->queue_thread = false;
return ret;
@ -142,7 +143,8 @@ void LFBTextBuffer::SpawnThreads()
return;
Process* kernel_process = Scheduler::GetKernelProcess();
queue_thread = true; // Visible to new thread.
if ( !RunKernelThread(kernel_process, LFBTextBuffer__RenderThread, this) )
if ( !RunKernelThread(kernel_process, LFBTextBuffer__RenderThread, this,
"console") )
queue_thread = false;
}

9
kernel/process.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen.
* Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -141,6 +141,10 @@ Process::Process()
threads_not_exiting_count = 0;
threads_exiting = false;
futex_lock = KTHREAD_MUTEX_INITIALIZER;
futex_first_waiting = NULL;
futex_last_waiting = NULL;
segments = NULL;
segments_used = 0;
segments_length = 0;
@ -1550,7 +1554,8 @@ pid_t sys_tfork(int flags, struct tfork* user_regs)
// If the thread could not be created, make the process commit suicide
// in a manner such that we don't wait for its zombie.
Thread* thread = CreateKernelThread(child_process, &cpuregs);
Thread* thread = CreateKernelThread(child_process, &cpuregs,
making_process ? "main" : "second");
process_family_lock_lock.Reset();
if ( !thread )
{

70
kernel/scheduler.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, 2012, 2013, 2014, 2015 Jonas 'Sortie' Termansen.
* Copyright (c) 2011, 2012, 2013, 2014, 2015, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -265,10 +265,17 @@ static void SwitchRegisters(struct interrupt_context* intctx,
current_thread = next;
}
static Thread* idle_thread;
static Thread* first_runnable_thread;
static Thread* true_current_thread;
static Process* init_process;
static void SwitchThread(struct interrupt_context* intctx,
Thread* old_thread,
Thread* new_thread)
{
assert(new_thread->state == ThreadState::RUNNABLE ||
new_thread == idle_thread);
SwitchRegisters(intctx, old_thread, new_thread);
if ( intctx->signal_pending && InUserspace(intctx) )
{
@ -290,14 +297,11 @@ static void SwitchThread(struct interrupt_context* intctx,
}
}
static Thread* idle_thread;
static Thread* first_runnable_thread;
static Thread* true_current_thread;
static Process* init_process;
static Thread* FindRunnableThreadWithSystemTid(uintptr_t system_tid)
{
Thread* begun_thread = current_thread;
Thread* begun_thread = first_runnable_thread;
if ( !begun_thread )
return NULL;
Thread* iter = begun_thread;
do
{
@ -329,8 +333,6 @@ static Thread* PopNextThread(bool yielded)
result = idle_thread;
}
true_current_thread = result;
return result;
}
@ -339,7 +341,10 @@ void SwitchTo(struct interrupt_context* intctx, Thread* new_thread)
Thread* old_thread = CurrentThread();
if ( new_thread == old_thread )
return;
first_runnable_thread = old_thread;
if ( new_thread->state != ThreadState::RUNNABLE )
return;
if ( old_thread != idle_thread )
first_runnable_thread = old_thread;
true_current_thread = new_thread;
SwitchThread(intctx, old_thread, new_thread);
}
@ -348,6 +353,7 @@ static void RealSwitch(struct interrupt_context* intctx, bool yielded)
{
Thread* old_thread = CurrentThread();
Thread* new_thread = PopNextThread(yielded);
true_current_thread = new_thread;
SwitchThread(intctx, old_thread, new_thread);
}
@ -358,7 +364,28 @@ void Switch(struct interrupt_context* intctx)
void InterruptYieldCPU(struct interrupt_context* intctx, void* /*user*/)
{
RealSwitch(intctx, true);
if ( current_thread->yield_operation == YIELD_OPERATION_NONE )
RealSwitch(intctx, true);
else if ( current_thread->yield_operation == YIELD_OPERATION_WAIT_FUTEX )
{
if ( !current_thread->futex_woken &&
!current_thread->timer_woken &&
!asm_signal_is_pending )
{
SetThreadState(current_thread, ThreadState::FUTEX_WAITING);
RealSwitch(intctx, false);
}
}
else if ( current_thread->yield_operation ==
YIELD_OPERATION_WAIT_FUTEX_SIGNAL )
{
if ( !current_thread->futex_woken &&
!current_thread->timer_woken )
{
SetThreadState(current_thread, ThreadState::FUTEX_WAITING);
RealSwitch(intctx, false);
}
}
}
void ThreadExitCPU(struct interrupt_context* intctx, void* /*user*/)
@ -395,9 +422,13 @@ Process* GetKernelProcess()
return idle_thread->process;
}
void SetThreadState(Thread* thread, ThreadState state)
void SetThreadState(Thread* thread, ThreadState state, bool wake_only)
{
bool wasenabled = Interrupt::SetEnabled(false);
bool was_enabled = Interrupt::SetEnabled(false);
// Avoid transitioning to the RUNNABLE state from unintended states.
if ( wake_only && thread->state != ThreadState::FUTEX_WAITING )
state = thread->state;
// Remove the thread from the list of runnable threads.
if ( thread->state == ThreadState::RUNNABLE &&
@ -432,7 +463,14 @@ void SetThreadState(Thread* thread, ThreadState state)
assert(thread->state != ThreadState::RUNNABLE || thread->scheduler_list_prev);
assert(thread->state != ThreadState::RUNNABLE || thread->scheduler_list_next);
Interrupt::SetEnabled(wasenabled);
Interrupt::SetEnabled(was_enabled);
}
void SetSignalPending(Thread* thread, unsigned long is_pending)
{
thread->registers.signal_pending = is_pending;
if ( is_pending )
Scheduler::SetThreadState(thread, ThreadState::RUNNABLE, true);
}
ThreadState GetThreadState(Thread* thread)
@ -457,14 +495,14 @@ namespace Scheduler {
void ScheduleTrueThread()
{
bool wasenabled = Interrupt::SetEnabled(false);
bool was_enabled = Interrupt::SetEnabled(false);
if ( true_current_thread != current_thread )
{
current_thread->yield_to_tid = 0;
first_runnable_thread = true_current_thread;
kthread_yield();
}
Interrupt::SetEnabled(wasenabled);
Interrupt::SetEnabled(was_enabled);
}
} // namespace Scheduler

2
kernel/signal.cpp
View File

@ -100,7 +100,7 @@ void UpdatePendingSignals(Thread* thread) // thread->process->signal_lock held
if ( thread == CurrentThread() )
asm_signal_is_pending = is_pending;
else
thread->registers.signal_pending = is_pending;
Scheduler::SetSignalPending(thread, is_pending);
}
void Thread::DoUpdatePendingSignal()

3
kernel/syscall.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016 Jonas 'Sortie' Termansen.
* Copyright (c) 2011-2016, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -202,6 +202,7 @@ void* syscall_list[SYSCALL_MAX_NUM + 1] =
[SYSCALL_SOCKET] = (void*) sys_socket,
[SYSCALL_GETDNSCONFIG] = (void*) sys_getdnsconfig,
[SYSCALL_SETDNSCONFIG] = (void*) sys_setdnsconfig,
[SYSCALL_FUTEX] = (void*) sys_futex,
[SYSCALL_MAX_NUM] = (void*) sys_bad_syscall,
};
} /* extern "C" */

163
kernel/thread.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011-2016, 2018 Jonas 'Sortie' Termansen.
* Copyright (c) 2011-2016, 2018, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -21,16 +21,21 @@
#include <assert.h>
#include <errno.h>
#include <limits.h>
#include <signal.h>
#include <stdlib.h>
#include <string.h>
#include <timespec.h>
#include <sortix/clock.h>
#include <sortix/exit.h>
#include <sortix/futex.h>
#include <sortix/mman.h>
#include <sortix/signal.h>
#include <sortix/kernel/copy.h>
#include <sortix/kernel/interrupt.h>
#include <sortix/kernel/ioctx.h>
#include <sortix/kernel/kernel.h>
#include <sortix/kernel/kthread.h>
#include <sortix/kernel/memorymanagement.h>
@ -77,6 +82,7 @@ void FreeThread(Thread* thread)
Thread::Thread()
{
assert(!((uintptr_t) registers.fpuenv & 0xFUL));
name = "";
system_tid = (uintptr_t) this;
yield_to_tid = 0;
id = 0; // TODO: Make a thread id.
@ -105,6 +111,11 @@ Thread::Thread()
// execute_clock initialized in member constructor.
// system_clock initialized in member constructor.
Time::InitializeThreadClocks(this);
futex_address = 0;
futex_woken = false;
futex_prev_waiting = NULL;
futex_next_waiting = NULL;
yield_operation = YIELD_OPERATION_NONE;
}
Thread::~Thread()
@ -116,7 +127,9 @@ Thread::~Thread()
delete[] (uint8_t*) kernelstackpos;
}
Thread* CreateKernelThread(Process* process, struct thread_registers* regs)
Thread* CreateKernelThread(Process* process,
struct thread_registers* regs,
const char* name)
{
assert(process && regs && process->addrspace);
@ -142,6 +155,7 @@ Thread* CreateKernelThread(Process* process, struct thread_registers* regs)
Thread* thread = AllocateThread();
if ( !thread )
return NULL;
thread->name = name;
memcpy(&thread->registers, regs, sizeof(struct thread_registers));
@ -254,7 +268,7 @@ static void SetupKernelThreadRegs(struct thread_registers* regs,
}
Thread* CreateKernelThread(Process* process, void (*entry)(void*), void* user,
size_t stacksize)
const char* name, size_t stacksize)
{
const size_t DEFAULT_KERNEL_STACK_SIZE = 8 * 1024UL;
if ( !stacksize )
@ -264,9 +278,10 @@ Thread* CreateKernelThread(Process* process, void (*entry)(void*), void* user,
return NULL;
struct thread_registers regs;
SetupKernelThreadRegs(&regs, process, entry, user, (uintptr_t) stack, stacksize);
SetupKernelThreadRegs(&regs, process, entry, user, (uintptr_t) stack,
stacksize);
Thread* thread = CreateKernelThread(process, &regs);
Thread* thread = CreateKernelThread(process, &regs, name);
if ( !thread ) { delete[] stack; return NULL; }
thread->kernelstackpos = (uintptr_t) stack;
@ -276,9 +291,10 @@ Thread* CreateKernelThread(Process* process, void (*entry)(void*), void* user,
return thread;
}
Thread* CreateKernelThread(void (*entry)(void*), void* user, size_t stacksize)
Thread* CreateKernelThread(void (*entry)(void*), void* user, const char* name,
size_t stacksize)
{
return CreateKernelThread(CurrentProcess(), entry, user, stacksize);
return CreateKernelThread(CurrentProcess(), entry, user, name, stacksize);
}
void StartKernelThread(Thread* thread)
@ -286,9 +302,10 @@ void StartKernelThread(Thread* thread)
Scheduler::SetThreadState(thread, ThreadState::RUNNABLE);
}
Thread* RunKernelThread(Process* process, struct thread_registers* regs)
Thread* RunKernelThread(Process* process, struct thread_registers* regs,
const char* name)
{
Thread* thread = CreateKernelThread(process, regs);
Thread* thread = CreateKernelThread(process, regs, name);
if ( !thread )
return NULL;
StartKernelThread(thread);
@ -296,18 +313,19 @@ Thread* RunKernelThread(Process* process, struct thread_registers* regs)
}
Thread* RunKernelThread(Process* process, void (*entry)(void*), void* user,
size_t stacksize)
const char* name, size_t stacksize)
{
Thread* thread = CreateKernelThread(process, entry, user, stacksize);
Thread* thread = CreateKernelThread(process, entry, user, name, stacksize);
if ( !thread )
return NULL;
StartKernelThread(thread);
return thread;
}
Thread* RunKernelThread(void (*entry)(void*), void* user, size_t stacksize)
Thread* RunKernelThread(void (*entry)(void*), void* user, const char* name,
size_t stacksize)
{
Thread* thread = CreateKernelThread(entry, user, stacksize);
Thread* thread = CreateKernelThread(entry, user, name, stacksize);
if ( !thread )
return NULL;
StartKernelThread(thread);
@ -323,7 +341,8 @@ int sys_exit_thread(int requested_exit_code,
EXIT_THREAD_ZERO |
EXIT_THREAD_TLS_UNMAP |
EXIT_THREAD_PROCESS |
EXIT_THREAD_DUMP_CORE) )
EXIT_THREAD_DUMP_CORE |
EXIT_THREAD_FUTEX_WAKE) )
return errno = EINVAL, -1;
if ( (flags & EXIT_THREAD_ONLY_IF_OTHERS) && (flags & EXIT_THREAD_PROCESS) )
@ -395,6 +414,9 @@ int sys_exit_thread(int requested_exit_code,
if ( flags & EXIT_THREAD_ZERO )
ZeroUser(extended.zero_from, extended.zero_size);
if ( flags & EXIT_THREAD_FUTEX_WAKE )
sys_futex((int*) extended.zero_from, FUTEX_WAKE, 1, NULL);
if ( do_exit )
{
// Validate the requested exit code such that the process can't exit
@ -431,4 +453,117 @@ int sys_exit_thread(int requested_exit_code,
kthread_exit();
}
static void futex_timeout(Clock* /*clock*/, Timer* /*timer*/, void* ctx)
{
Thread* thread = (Thread*) ctx;
thread->timer_woken = true;
kthread_wake_futex(thread);
}
int sys_futex(int* user_address,
int op,
int value,
const struct timespec* user_timeout)
{
ioctx_t ctx; SetupKernelIOCtx(&ctx);
Thread* thread = CurrentThread();
Process* process = thread->process;
if ( FUTEX_GET_OP(op) == FUTEX_WAIT )
{
kthread_mutex_lock(&process->futex_lock);
thread->futex_address = (uintptr_t) user_address;
thread->futex_woken = false;
thread->futex_prev_waiting = process->futex_last_waiting;
thread->futex_next_waiting = NULL;
(process->futex_last_waiting ?
process->futex_last_waiting->futex_next_waiting :
process->futex_first_waiting) = thread;
process->futex_last_waiting = thread;
kthread_mutex_unlock(&process->futex_lock);
thread->timer_woken = false;
Timer timer;
if ( user_timeout )
{
clockid_t clockid = FUTEX_GET_CLOCK(op);
bool absolute = op & FUTEX_ABSOLUTE;
struct timespec timeout;
if ( !CopyFromUser(&timeout, user_timeout, sizeof(timeout)) )
return -1;
if ( !timespec_is_canonical(timeout) )
return errno = EINVAL, -1;
Clock* clock = Time::GetClock(clockid);
timer.Attach(clock);
struct itimerspec timerspec;
timerspec.it_value = timeout;
timerspec.it_interval.tv_sec = 0;
timerspec.it_interval.tv_nsec = 0;
int timer_flags = (absolute ? TIMER_ABSOLUTE : 0) |
TIMER_FUNC_INTERRUPT_HANDLER;
timer.Set(&timerspec, NULL, timer_flags, futex_timeout, thread);
}
int result = 0;
int current;
if ( !ReadAtomicFromUser(&current, user_address) )
result = -1;
else if ( current != value )
{
errno = EAGAIN;
result = -1;
}
else
kthread_wait_futex_signal();
if ( user_timeout )
timer.Cancel();
kthread_mutex_lock(&process->futex_lock);
if ( result == 0 && !thread->futex_woken )
{
if ( Signal::IsPending() )
{
errno = EINTR;
result = -1;
}
else if ( thread->timer_woken )
{
errno = ETIMEDOUT;
result = -1;
}
}
thread->futex_address = 0;
thread->futex_woken = false;
(thread->futex_prev_waiting ?
thread->futex_prev_waiting->futex_next_waiting :
process->futex_first_waiting) = thread->futex_next_waiting;
(thread->futex_next_waiting ?
thread->futex_next_waiting->futex_prev_waiting :
process->futex_last_waiting) = thread->futex_prev_waiting;
thread->futex_prev_waiting = NULL;
thread->futex_next_waiting = NULL;
kthread_mutex_unlock(&process->futex_lock);
return result;
}
else if ( FUTEX_GET_OP(op) == FUTEX_WAKE )
{
kthread_mutex_lock(&process->futex_lock);
int result = 0;
for ( Thread* waiter = process->futex_first_waiting;
0 < value && waiter;
waiter = waiter->futex_next_waiting )
{
if ( waiter->futex_address == (uintptr_t) user_address )
{
waiter->futex_woken = true;
kthread_wake_futex(waiter);
if ( value != INT_MAX )
value--;
if ( result != INT_MAX )
result++;
}
}
kthread_mutex_unlock(&process->futex_lock);
return result;
}
else
return errno = EINVAL, -1;
}
} // namespace Sortix

84
kernel/x64/kthread.S
View File

@ -1,84 +0,0 @@
/*
* Copyright (c) 2012 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*
* x64/kthread.S
* Utilities and synchronization mechanisms for x64 kernel threads.
*/
.section .text
.global kthread_mutex_trylock
.type kthread_mutex_trylock, @function
kthread_mutex_trylock:
pushq %rbp
movq %rsp, %rbp
movl $-1, %eax
xchgl (%rdi), %eax
not %eax
leaveq
retq
.size kthread_mutex_trylock, . - kthread_mutex_trylock
.global kthread_mutex_lock
.type kthread_mutex_lock, @function
kthread_mutex_lock:
pushq %rbp
movq %rsp, %rbp
kthread_mutex_lock_retry:
movl $-1, %eax
xchgl (%rdi), %eax
testl %eax, %eax
jnz kthread_mutex_lock_failed
leaveq
retq
kthread_mutex_lock_failed:
int $0x81 # Yield the CPU.
jmp kthread_mutex_lock_retry
.size kthread_mutex_lock, . - kthread_mutex_lock
.global kthread_mutex_lock_signal
.type kthread_mutex_lock_signal, @function
kthread_mutex_lock_signal:
pushq %rbp
movq %rsp, %rbp
kthread_mutex_lock_signal_retry:
movq asm_signal_is_pending, %rax
testq %rax, %rax
jnz kthread_mutex_lock_signal_pending
movl $-1, %eax
xchgl (%rdi), %eax
testl %eax, %eax
jnz kthread_mutex_lock_signal_failed
inc %eax
kthread_mutex_lock_signal_out:
leaveq
retq
kthread_mutex_lock_signal_failed:
int $0x81 # Yield the CPU.
jmp kthread_mutex_lock_signal_retry
kthread_mutex_lock_signal_pending:
xorl %eax, %eax
jmp kthread_mutex_lock_signal_out
.size kthread_mutex_lock_signal, . - kthread_mutex_lock_signal
.global kthread_mutex_unlock
.type kthread_mutex_unlock, @function
kthread_mutex_unlock:
pushq %rbp
movq %rsp, %rbp
movl $0, (%rdi)
leaveq
retq
.size kthread_mutex_unlock, . - kthread_mutex_unlock

87
kernel/x86/kthread.S
View File

@ -1,87 +0,0 @@
/*
* Copyright (c) 2012 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*
* x86/kthread.S
* Utilities and synchronization mechanisms for x86 kernel threads.
*/
.section .text
.global kthread_mutex_trylock
.type kthread_mutex_trylock, @function
kthread_mutex_trylock:
pushl %ebp
movl %esp, %ebp
movl 8(%ebp), %edx
movl $-1, %eax
xchgl (%edx), %eax
not %eax
leavel
retl
.size kthread_mutex_trylock, . - kthread_mutex_trylock
.global kthread_mutex_lock
.type kthread_mutex_lock, @function
kthread_mutex_lock:
pushl %ebp
movl %esp, %ebp
movl 8(%ebp), %edx
kthread_mutex_lock_retry:
movl $-1, %eax
xchgl (%edx), %eax
testl %eax, %eax
jnz kthread_mutex_lock_failed
leavel
retl
kthread_mutex_lock_failed:
int $0x81 # Yield the CPU.
jmp kthread_mutex_lock_retry
.size kthread_mutex_lock, . - kthread_mutex_lock
.global kthread_mutex_lock_signal
.type kthread_mutex_lock_signal, @function
kthread_mutex_lock_signal:
pushl %ebp
movl %esp, %ebp
movl 8(%ebp), %edx
kthread_mutex_lock_signal_retry:
movl asm_signal_is_pending, %eax
testl %eax, %eax
jnz kthread_mutex_lock_signal_pending
movl $-1, %eax
xchgl (%edx), %eax
testl %eax, %eax
jnz kthread_mutex_lock_signal_failed
inc %eax
kthread_mutex_lock_signal_out:
leavel
retl
kthread_mutex_lock_signal_failed:
int $0x81 # Yield the CPU.
jmp kthread_mutex_lock_signal_retry
kthread_mutex_lock_signal_pending:
xorl %eax, %eax
jmp kthread_mutex_lock_signal_out
.global kthread_mutex_unlock
.type kthread_mutex_unlock, @function
kthread_mutex_unlock:
pushl %ebp
movl %esp, %ebp
movl 8(%ebp), %edx
movl $0, (%edx)
leavel
retl
.size kthread_mutex_lock_signal, . - kthread_mutex_lock_signal

1
libc/Makefile
View File

@ -561,6 +561,7 @@ sys/ioctl/ioctl.o \
sys/kernelinfo/kernelinfo.o \
syslog/closelog.o \
syslog/connectlog.o \
sys/futex/futex.o \
syslog/openlog.o \
syslog/setlogmask.o \
syslog/syslog.o \

64
libc/include/__/pthread.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2014, 2017 Jonas 'Sortie' Termansen.
* Copyright (c) 2013, 2014, 2017, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -46,40 +46,12 @@ typedef int __pthread_barrier_t;
typedef int __pthread_barrierattr_t;
#if defined(__is_sortix_libc)
typedef struct
{
struct pthread_cond_elem* first;
struct pthread_cond_elem* last;
__clock_t clock;
} __pthread_cond_t;
#else
typedef struct
{
void* __pthread_first;
void* __pthread_last;
__clock_t __pthread_clock;
} __pthread_cond_t;
#endif
#if defined(__is_sortix_libc)
typedef struct
{
__clock_t clock;
} __pthread_condattr_t;
#else
typedef struct
{
__clock_t __pthread_clock;
} __pthread_condattr_t;
#endif
typedef __SIZE_TYPE__ __pthread_key_t;
#if defined(__is_sortix_libc)
typedef struct
{
unsigned long lock;
int lock;
unsigned long type;
unsigned long owner;
unsigned long recursion;
@ -87,7 +59,7 @@ typedef struct
#else
typedef struct
{
unsigned long __pthread_lock;
int __pthread_lock;
unsigned long __pthread_type;
unsigned long __pthread_owner;
unsigned long __pthread_recursion;
@ -106,6 +78,36 @@ typedef struct
} __pthread_mutexattr_t;
#endif
#if defined(__is_sortix_libc)
typedef struct
{
__pthread_mutex_t lock;
struct pthread_cond_elem* first;
struct pthread_cond_elem* last;
__clockid_t clock;
} __pthread_cond_t;
#else
typedef struct
{
__pthread_mutex_t __pthread_lock;
void* __pthread_first;
void* __pthread_last;
__clockid_t __pthread_clock;
} __pthread_cond_t;
#endif
#if defined(__is_sortix_libc)
typedef struct
{
__clockid_t clock;
} __pthread_condattr_t;
#else
typedef struct
{
__clockid_t __pthread_clock;
} __pthread_condattr_t;
#endif
#if defined(__is_sortix_libc)
typedef struct
{

11
libc/include/pthread.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -150,18 +150,21 @@ struct pthread
struct pthread_cond_elem
{
struct pthread_cond_elem* next;
volatile unsigned long woken;
struct pthread_cond_elem* prev;
int woken;
};
#endif
#define PTHREAD_COND_INITIALIZER { NULL, NULL, CLOCK_REALTIME }
#define PTHREAD_COND_INITIALIZER { PTHREAD_NORMAL_MUTEX_INITIALIZER_NP, NULL, \
NULL, CLOCK_REALTIME }
#define PTHREAD_MUTEX_INITIALIZER { 0, PTHREAD_MUTEX_DEFAULT, 0, 0 }
#define PTHREAD_RWLOCK_INITIALIZER { PTHREAD_COND_INITIALIZER, \
PTHREAD_COND_INITIALIZER, \
PTHREAD_MUTEX_INITIALIZER, 0, 0, 0, 0 }
#define PTHREAD_NORMAL_MUTEX_INITIALIZER_NP { 0, PTHREAD_MUTEX_NORMAL, 0, 0 }
#define PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP { 0, PTHREAD_MUTEX_RECURSIVE, 0, 0 }
#define PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP { 0, PTHREAD_MUTEX_RECURSIVE, \
0, 0 }
#define PTHREAD_ONCE_INIT { PTHREAD_NORMAL_MUTEX_INITIALIZER_NP, 0 }

4
libc/include/semaphore.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -32,8 +32,10 @@ typedef struct
{
#if defined(__is_sortix_libc)
int value;
int waiters;
#else
int __value;
int __waiters;
#endif
} sem_t;

30
libc/include/sys/futex.h Normal file
View File

@ -0,0 +1,30 @@
/*
* Copyright (c) 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*
* sys/futex.h
* Fast userspace mutexes.
*/
#ifndef _SYS_FUTEX_H
#define _SYS_FUTEX_H 1
#include <sys/cdefs.h>
#include <sortix/futex.h>
#include <sortix/timespec.h>
int futex(int*, int, int, const struct timespec*);
#endif

22
libc/pthread/pthread_cond_broadcast.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013 Jonas 'Sortie' Termansen.
* Copyright (c) 2013, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -17,13 +17,27 @@
* Broadcasts a condition.
*/
#include <sys/futex.h>
#include <stddef.h>
#include <pthread.h>
int pthread_cond_broadcast(pthread_cond_t* cond)
{
int ret;
pthread_mutex_lock(&cond->lock);
while ( cond->first )
if ( (ret = pthread_cond_signal(cond)) )
return ret;
{
struct pthread_cond_elem* elem = cond->first;
if ( elem->next )
elem->next->prev = elem->prev;
else
cond->last = elem->prev;
cond->first = elem->next;
elem->next = NULL;
elem->prev = NULL;
__atomic_store_n(&elem->woken, 1, __ATOMIC_SEQ_CST);
futex(&elem->woken, FUTEX_WAKE, 1, NULL);
}
pthread_mutex_unlock(&cond->lock);
return 0;
}

4
libc/pthread/pthread_cond_destroy.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013 Jonas 'Sortie' Termansen.
* Copyright (c) 2013, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -21,6 +21,6 @@
int pthread_cond_destroy(pthread_cond_t* restrict cond)
{
(void) cond;
pthread_mutex_destroy(&cond->lock);
return 0;
}

3
libc/pthread/pthread_cond_init.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -29,6 +29,7 @@ int pthread_cond_init(pthread_cond_t* restrict cond,
attr = &default_attr;
}
pthread_mutex_init(&cond->lock, NULL);
cond->first = NULL;
cond->last = NULL;
cond->clock = attr->clock;

27
libc/pthread/pthread_cond_signal.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013 Jonas 'Sortie' Termansen.
* Copyright (c) 2013, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -17,16 +17,27 @@
* Signals a condition.
*/
#include <sys/futex.h>
#include <stddef.h>
#include <pthread.h>
int pthread_cond_signal(pthread_cond_t* cond)
{
struct pthread_cond_elem* elem = cond->first;
if ( !elem )
return 0;
if ( !(cond->first = elem->next) )
cond->last = NULL;
elem->next = NULL;
elem->woken = 1;
pthread_mutex_lock(&cond->lock);
if ( cond->first )
{
struct pthread_cond_elem* elem = cond->first;
if ( elem->next )
elem->next->prev = elem->prev;
else
cond->last = elem->prev;
cond->first = elem->next;
elem->next = NULL;
elem->prev = NULL;
__atomic_store_n(&elem->woken, 1, __ATOMIC_SEQ_CST);
futex(&elem->woken, FUTEX_WAKE, 1, NULL);
}
pthread_mutex_unlock(&cond->lock);
return 0;
}

48
libc/pthread/pthread_cond_timedwait.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -17,35 +17,51 @@
* Waits on a condition or until a timeout happens.
*/
#include <sys/futex.h>
#include <errno.h>
#include <pthread.h>
#include <sched.h>
#include <time.h>
#include <timespec.h>
#include <unistd.h>
int pthread_cond_timedwait(pthread_cond_t* restrict cond,
pthread_mutex_t* restrict mutex,
const struct timespec* restrict abstime)
{
struct pthread_cond_elem elem;
pthread_mutex_lock(&cond->lock);
elem.next = NULL;
elem.prev = cond->last;
elem.woken = 0;
if ( cond->last )
cond->last->next = &elem;
if ( !cond->last )
if ( !cond->first )
cond->first = &elem;
cond->last = &elem;
while ( !elem.woken )
pthread_mutex_unlock(&cond->lock);
pthread_mutex_unlock(mutex);
int op = FUTEX_WAIT | FUTEX_ABSOLUTE | FUTEX_CLOCK(cond->clock);
int result = 0;
while ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) &&
futex(&elem.woken, op, 0, abstime) < 0 )
{
struct timespec now;
if ( clock_gettime(cond->clock, &now) < 0 )
return errno;
if ( timespec_le(*abstime, now) )
return errno = ETIMEDOUT;
pthread_mutex_unlock(mutex);
sched_yield();
pthread_mutex_lock(mutex);
if ( errno == EINTR )
continue;
if ( errno != EAGAIN )
result = errno;
break;
}
return 0;
pthread_mutex_lock(mutex);
pthread_mutex_lock(&cond->lock);
if ( !__atomic_load_n(&elem.woken, __ATOMIC_SEQ_CST) )
{
if ( elem.next )
elem.next->prev = elem.prev;
else
cond->last = elem.prev;
if ( elem.prev )
elem.prev->next = elem.next;
else
cond->first = elem.next;
}
pthread_mutex_unlock(&cond->lock);
return result;
}

23
libc/pthread/pthread_cond_wait.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -17,28 +17,11 @@
* Waits on a condition.
*/
#include <stddef.h>
#include <pthread.h>
#include <sched.h>
#include <time.h>
#include <timespec.h>
#include <unistd.h>
int pthread_cond_wait(pthread_cond_t* restrict cond,
pthread_mutex_t* restrict mutex)
{
struct pthread_cond_elem elem;
elem.next = NULL;
elem.woken = 0;
if ( cond->last )
cond->last->next = &elem;
if ( !cond->last )
cond->first = &elem;
cond->last = &elem;
while ( !elem.woken )
{
pthread_mutex_unlock(mutex);
sched_yield();
pthread_mutex_lock(mutex);
}
return 0;
return pthread_cond_timedwait(cond, mutex, NULL);
}

4
libc/pthread/pthread_exit.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -62,7 +62,7 @@ void pthread_exit(void* return_value)
{
extended.zero_from = &thread->join_lock.lock;
extended.zero_size = sizeof(thread->join_lock.lock);
exit_flags |= EXIT_THREAD_ZERO;
exit_flags |= EXIT_THREAD_ZERO | EXIT_THREAD_FUTEX_WAKE;
}
else
{

43
libc/pthread/pthread_mutex_lock.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -17,22 +17,45 @@
* Locks a mutex.
*/
#include <pthread.h>
#include <sched.h>
#include <time.h>
#include <timespec.h>
#include <sys/futex.h>
static const unsigned long UNLOCKED_VALUE = 0;
static const unsigned long LOCKED_VALUE = 1;
#include <errno.h>
#include <limits.h>
#include <pthread.h>
#include <stdbool.h>
static const int UNLOCKED = 0;
static const int LOCKED = 1;
static const int CONTENDED = 2;
int pthread_mutex_lock(pthread_mutex_t* mutex)
{
while ( !__sync_bool_compare_and_swap(&mutex->lock, UNLOCKED_VALUE, LOCKED_VALUE) )
int state = UNLOCKED;
int desired = LOCKED;
while ( !__atomic_compare_exchange_n(&mutex->lock, &state, desired, false,
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) )
{
if ( mutex->type == PTHREAD_MUTEX_RECURSIVE &&
(pthread_t) mutex->owner == pthread_self() )
return mutex->recursion++, 0;
sched_yield();
{
if ( mutex->recursion == ULONG_MAX )
return errno = EAGAIN;
mutex->recursion++;
return 0;
}
if ( state == LOCKED &&
!__atomic_compare_exchange_n(&mutex->lock, &state, CONTENDED,
false, __ATOMIC_SEQ_CST,
__ATOMIC_SEQ_CST) )
{
state = UNLOCKED;
continue;
}
desired = CONTENDED;
if ( futex(&mutex->lock, FUTEX_WAIT, CONTENDED, NULL) < 0 &&
errno != EAGAIN && errno != EINTR )
return errno;
state = UNLOCKED;
}
mutex->owner = (unsigned long) pthread_self();
mutex->recursion = 0;

19
libc/pthread/pthread_mutex_trylock.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -18,18 +18,27 @@
*/
#include <errno.h>
#include <limits.h>
#include <pthread.h>
#include <stdbool.h>
static const unsigned long UNLOCKED_VALUE = 0;
static const unsigned long LOCKED_VALUE = 1;
static const int UNLOCKED = 0;
static const int LOCKED = 1;
int pthread_mutex_trylock(pthread_mutex_t* mutex)
{
if ( !__sync_bool_compare_and_swap(&mutex->lock, UNLOCKED_VALUE, LOCKED_VALUE) )
int state = UNLOCKED;
if ( !__atomic_compare_exchange_n(&mutex->lock, &state, LOCKED, false,
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) )
{
if ( mutex->type == PTHREAD_MUTEX_RECURSIVE &&
(pthread_t) mutex->owner == pthread_self() )
return mutex->recursion++, 0;
{
if ( mutex->recursion == ULONG_MAX )
return errno = EAGAIN;
mutex->recursion++;
return 0;
}
return errno = EBUSY;
}
mutex->owner = (unsigned long) pthread_self();

20
libc/pthread/pthread_mutex_unlock.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2013, 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -17,16 +17,24 @@
* Unlocks a mutex.
*/
#include <pthread.h>
#include <sys/futex.h>
static const unsigned long UNLOCKED_VALUE = 0;
static const unsigned long LOCKED_VALUE = 1;
#include <pthread.h>
#include <stdbool.h>
static const int UNLOCKED = 0;
static const int CONTENDED = 2;
int pthread_mutex_unlock(pthread_mutex_t* mutex)
{
if ( mutex->type == PTHREAD_MUTEX_RECURSIVE && mutex->recursion )
return mutex->recursion--, 0;
{
mutex->recursion--;
return 0;
}
mutex->owner = 0;
mutex->lock = UNLOCKED_VALUE;
if ( __atomic_exchange_n(&mutex->lock, UNLOCKED,
__ATOMIC_SEQ_CST) == CONTENDED )
futex(&mutex->lock, FUTEX_WAKE, 1, NULL);
return 0;
}

5
libc/semaphore/sem_getvalue.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -21,6 +21,7 @@
int sem_getvalue(sem_t* restrict sem, int* restrict value_ptr)
{
*value_ptr = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST);
int value = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST);
*value_ptr = value == -1 ? 0 : value;
return 0;
}

6
libc/semaphore/sem_init.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -25,11 +25,9 @@ int sem_init(sem_t* sem, int pshared, unsigned int value)
{
if ( pshared )
return errno = ENOSYS, -1;
if ( (unsigned int) INT_MAX < value )
return errno = EINVAL, -1;
sem->value = (int) value;
sem->waiters = 0;
return 0;
}

21
libc/semaphore/sem_post.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -17,25 +17,28 @@
* Unlock a semaphore.
*/
#include <sys/futex.h>
#include <errno.h>
#include <limits.h>
#include <semaphore.h>
#include <stdbool.h>
#include <stddef.h>
int sem_post(sem_t* sem)
{
while ( true )
{
int old_value = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST);
if ( old_value == INT_MAX )
return errno = EOVERFLOW;
int new_value = old_value + 1;
if ( !__atomic_compare_exchange_n(&sem->value, &old_value, new_value,
false,
int old = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST);
if ( old == INT_MAX )
return errno = EOVERFLOW, -1;
int new = old == -1 ? 1 : old + 1;
int waiters = __atomic_load_n(&sem->waiters, __ATOMIC_SEQ_CST);
if ( !__atomic_compare_exchange_n(&sem->value, &old, new, false,
__ATOMIC_SEQ_CST, __ATOMIC_RELAXED) )
continue;
if ( old == -1 || waiters )
futex(&sem->value, FUTEX_WAKE, 1, NULL);
return 0;
}
}

64
libc/semaphore/sem_timedwait.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -14,57 +14,39 @@
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*
* semaphore/sem_timedwait.c
* Lock a semaphore.
* Lock a semaphore with a timeout.
*/
#include <sys/futex.h>
#include <errno.h>
#include <sched.h>
#include <semaphore.h>
#include <signal.h>
#include <stddef.h>
#include <stdbool.h>
#include <time.h>
#include <timespec.h>
int sem_timedwait(sem_t* restrict sem, const struct timespec* restrict abstime)
{
if ( sem_trywait(sem) == 0 )
return 0;
if ( errno != EAGAIN )
return -1;
sigset_t old_set_mask;
sigset_t old_set_allowed;
sigset_t all_signals;
sigfillset(&all_signals);
sigprocmask(SIG_SETMASK, &all_signals, &old_set_mask);
signotset(&old_set_allowed, &old_set_mask);
while ( sem_trywait(sem) != 0 )
while ( true )
{
// TODO: Using CLOCK_REALTIME for this is bad as it is not monotonic. We
// need to enchance the semaphore API so a better clock can be
// used instead.
if ( errno == EAGAIN )
int old = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST);
int new = old != -1 ? old - 1 : -1;
bool waiting = new == -1;
if ( waiting )
__atomic_add_fetch(&sem->waiters, 1, __ATOMIC_SEQ_CST);
if ( old != new &&
!__atomic_compare_exchange_n(&sem->value, &old, new, false,
__ATOMIC_SEQ_CST, __ATOMIC_RELAXED) )
{
struct timespec now;
clock_gettime(CLOCK_REALTIME, &now);
if ( timespec_le(*abstime, now) )
errno = ETIMEDOUT;
if ( waiting )
__atomic_sub_fetch(&sem->waiters, 1, __ATOMIC_SEQ_CST);
continue;
}
if ( errno == EAGAIN && sigpending(&old_set_allowed) )
errno = EINTR;
if ( errno != EAGAIN )
{
sigprocmask(SIG_SETMASK, &old_set_mask, NULL);
if ( !waiting )
return 0;
int op = FUTEX_WAIT | FUTEX_ABSOLUTE | FUTEX_CLOCK(CLOCK_REALTIME);
int ret = futex(&sem->value, op, -1, abstime);
__atomic_sub_fetch(&sem->waiters, 1, __ATOMIC_SEQ_CST);
if ( ret < 0 && errno != EAGAIN )
return -1;
}
sched_yield();
}
sigprocmask(SIG_SETMASK, &old_set_mask, NULL);
return 0;
}

12
libc/semaphore/sem_trywait.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -14,7 +14,7 @@
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*
* semaphore/sem_trywait.c
* Lock a semaphore.
* Try to lock a semaphore.
*/
#include <errno.h>
@ -23,11 +23,11 @@
int sem_trywait(sem_t* sem)
{
int old_value = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST);
if ( old_value <= 0 )
int old = __atomic_load_n(&sem->value, __ATOMIC_SEQ_CST);
if ( old <= 0 )
return errno = EAGAIN, -1;
int new_value = old_value - 1;
if ( !__atomic_compare_exchange_n(&sem->value, &old_value, new_value, false,
int new = old != -1 ? old - 1 : -1;
if ( !__atomic_compare_exchange_n(&sem->value, &old, new, false,
__ATOMIC_SEQ_CST, __ATOMIC_RELAXED) )
return errno = EAGAIN, -1;
return 0;

35
libc/semaphore/sem_wait.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014 Jonas 'Sortie' Termansen.
* Copyright (c) 2014, 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@ -17,41 +17,10 @@
* Lock a semaphore.
*/
#include <errno.h>
#include <sched.h>
#include <semaphore.h>
#include <signal.h>
#include <stddef.h>
int sem_wait(sem_t* sem)
{
if ( sem_trywait(sem) == 0 )
return 0;
if ( errno != EAGAIN )
return -1;
sigset_t old_set_mask;
sigset_t old_set_allowed;
sigset_t all_signals;
sigfillset(&all_signals);
sigprocmask(SIG_SETMASK, &all_signals, &old_set_mask);
signotset(&old_set_allowed, &old_set_mask);
while ( sem_trywait(sem) != 0 )
{
if ( errno == EAGAIN && sigpending(&old_set_allowed) )
errno = EINTR;
if ( errno != EAGAIN )
{
sigprocmask(SIG_SETMASK, &old_set_mask, NULL);
return -1;
}
sched_yield();
}
sigprocmask(SIG_SETMASK, &old_set_mask, NULL);
return 0;
return sem_timedwait(sem, NULL);
}

29
libc/sys/futex/futex.c Normal file
View File

@ -0,0 +1,29 @@
/*
* Copyright (c) 2021 Jonas 'Sortie' Termansen.
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*
* sys/futex/futex.c
* Fast userspace mutexes.
*/
#include <sys/futex.h>
#include <sys/syscall.h>
DEFN_SYSCALL4(int, sys_futex, SYSCALL_FUTEX, int*, int, int,
const struct timespec*);
int futex(int* address, int op, int value, const struct timespec* timeout)
{
return sys_futex(address, op, value, timeout);
}

10
share/man/man7/following-development.7
View File

@ -69,6 +69,16 @@ releasing Sortix x.y, foo." to allow the maintainer to easily
.Xr grep 1
for it after a release.
.Sh CHANGES
.Ss Implement threading primitives that truly sleep
The
.Xr futex 2
system system call for efficient thread waiting has been added.
The
.Xr exit_thread 2
system call has gained a
.Dv EXIT_THREAD_FUTEX_WAKE
flag for waking a single waiter on a futex.
This is a minor compatible ABI change.
.Ss Fix system upgrade leaking files
.Xr sysupgrade 8
and