qiurui/Centos-kernel-stream-10
1
0
Fork 0
mirror of https://gitlab.com/redhat/centos-stream/src/kernel/centos-stream-10.git synced 2026-09-09 00:07:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge: time: Update kernel/time code up to v6.18 with fixes

Browse Source 
MR: https://gitlab.com/redhat/centos-stream/src/kernel/centos-stream-10/-/merge_requests/2197

JIRA: https://issues.redhat.com/browse/RHEL-152433
MR: https://gitlab.com/redhat/centos-stream/src/kernel/centos-stream-10/-/merge_requests/2197

This MR backports most of the kernel/time commits up to v6.18 with fixes
beyond that. The major exception is some namespace related commits which
have to be done separately.

Signed-off-by: Waiman Long <longman@redhat.com>

Approved-by: Ricardo Robaina <rrobaina@redhat.com>
Approved-by: Tony Camuso <tcamuso@redhat.com>
Approved-by: Antoine Tenart <atenart@redhat.com>
Approved-by: Rafael Aquini <raquini@redhat.com>
Approved-by: Čestmír Kalina <ckalina@redhat.com>
Approved-by: Phil Auld <pauld@redhat.com>
Approved-by: CKI KWF Bot <cki-ci-bot+kwf-gitlab-com@redhat.com>

Merged-by: Alexandra Hájková <ahajkova@redhat.com>
This commit is contained in:
Alexandra Hájková
2026-03-24 16:16:20 +01:00
parent cf07652d37 f5a9854d69
commit 069611259f
79 changed files with 1137 additions and 467 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Documentation/ABI/stable/sysfs-kernel-time-aux-clocks
+5
View File
@@ -0,0 +1,5 @@
What: /sys/kernel/time/aux_clocks/<ID>/enable
Date: May 2025
Contact: Thomas Gleixner <tglx@linutronix.de>
Description:
Controls the enablement of auxiliary clock timekeepers.
arch/arm64/include/asm/vdso/vsyscall.h
+3 -4
View File
@@ -13,12 +13,11 @@
* Update the vDSO data page to keep in sync with kernel timekeeping.
*/
static __always_inline
void __arm64_update_vsyscall(struct vdso_time_data *vdata)
void __arch_update_vdso_clock(struct vdso_clock *vc)
{
vdata->clock_data[CS_HRES_COARSE].mask = VDSO_PRECISION_MASK;
vdata->clock_data[CS_RAW].mask = VDSO_PRECISION_MASK;
vc->mask = VDSO_PRECISION_MASK;
}
#define __arch_update_vsyscall __arm64_update_vsyscall
#define __arch_update_vdso_clock __arch_update_vdso_clock
/* The asm-generic header needs to be included after the definitions above */
#include <asm-generic/vdso/vsyscall.h>
block/blk-ioc.c
+1 -1
View File
@@ -286,7 +286,7 @@ out:
}
EXPORT_SYMBOL_GPL(set_task_ioprio);
int __copy_io(unsigned long clone_flags, struct task_struct *tsk)
int __copy_io(u64 clone_flags, struct task_struct *tsk)
{
struct io_context *ioc = current->io_context;
drivers/media/rc/pwm-ir-tx.c
+1 -4
View File
@@ -117,7 +117,6 @@ static int pwm_ir_tx_atomic(struct rc_dev *dev, unsigned int *txbuf,
static enum hrtimer_restart pwm_ir_timer(struct hrtimer *timer)
{
struct pwm_ir *pwm_ir = container_of(timer, struct pwm_ir, timer);
ktime_t now;
/*
* If we happen to hit an odd latency spike, loop through the
@@ -139,9 +138,7 @@ static enum hrtimer_restart pwm_ir_timer(struct hrtimer *timer)
hrtimer_add_expires_ns(timer, ns);
pwm_ir->txbuf_index++;
now = timer->base->get_time();
} while (hrtimer_get_expires_tv64(timer) < now);
} while (hrtimer_expires_remaining(timer) > 0);
return HRTIMER_RESTART;
}
fs/namespace.c
+1 -1
View File
@@ -3968,7 +3968,7 @@ static struct mnt_namespace *alloc_mnt_ns(struct user_namespace *user_ns, bool a
}
__latent_entropy
struct mnt_namespace *copy_mnt_ns(unsigned long flags, struct mnt_namespace *ns,
struct mnt_namespace *copy_mnt_ns(u64 flags, struct mnt_namespace *ns,
struct user_namespace *user_ns, struct fs_struct *new_fs)
{
struct mnt_namespace *new_ns;
include/asm-generic/vdso/vsyscall.h
+3 -3
View File
@@ -22,11 +22,11 @@ static __always_inline const struct vdso_rng_data *__arch_get_vdso_u_rng_data(vo
#endif /* CONFIG_GENERIC_VDSO_DATA_STORE */
#ifndef __arch_update_vsyscall
static __always_inline void __arch_update_vsyscall(struct vdso_time_data *vdata)
#ifndef __arch_update_vdso_clock
static __always_inline void __arch_update_vdso_clock(struct vdso_clock *vc)
{
}
#endif /* __arch_update_vsyscall */
#endif /* __arch_update_vdso_clock */
#ifndef __arch_sync_vdso_time_data
static __always_inline void __arch_sync_vdso_time_data(struct vdso_time_data *vdata)
include/linux/cgroup.h
+2 -2
View File
@@ -779,7 +779,7 @@ extern struct cgroup_namespace init_cgroup_ns;
void free_cgroup_ns(struct cgroup_namespace *ns);
struct cgroup_namespace *copy_cgroup_ns(unsigned long flags,
struct cgroup_namespace *copy_cgroup_ns(u64 flags,
struct user_namespace *user_ns,
struct cgroup_namespace *old_ns);
@@ -790,7 +790,7 @@ int cgroup_path_ns(struct cgroup *cgrp, char *buf, size_t buflen,
static inline void free_cgroup_ns(struct cgroup_namespace *ns) { }
static inline struct cgroup_namespace *
copy_cgroup_ns(unsigned long flags, struct user_namespace *user_ns,
copy_cgroup_ns(u64 flags, struct user_namespace *user_ns,
struct cgroup_namespace *old_ns)
{
return old_ns;
include/linux/cpumask.h
+12
View File
@@ -354,6 +354,18 @@ unsigned int cpumask_next_wrap(int n, const struct cpumask *src)
return find_next_bit_wrap(cpumask_bits(src), small_cpumask_bits, n + 1);
}
/**
* cpumask_random - get random cpu in *src.
* @src: cpumask pointer
*
* Return: random set bit, or >= nr_cpu_ids if @src is empty.
*/
static __always_inline
unsigned int cpumask_random(const struct cpumask *src)
{
return find_random_bit(cpumask_bits(src), nr_cpu_ids);
}
/**
* for_each_cpu - iterate over every cpu in a mask
* @cpu: the (optionally unsigned) integer iterator
include/linux/cred.h
+1 -1
View File
@@ -148,7 +148,7 @@ struct cred {
extern void __put_cred(struct cred *);
extern void exit_creds(struct task_struct *);
extern int copy_creds(struct task_struct *, unsigned long);
extern int copy_creds(struct task_struct *, u64);
extern const struct cred *get_task_cred(struct task_struct *);
extern struct cred *cred_alloc_blank(void);
extern struct cred *prepare_creds(void);
include/linux/delay.h
+4 -4
View File
@@ -68,7 +68,7 @@ void usleep_range_state(unsigned long min, unsigned long max,
* @min: Minimum time in microseconds to sleep
* @max: Maximum time in microseconds to sleep
*
* For basic information please refere to usleep_range_state().
* For basic information please refer to usleep_range_state().
*
* The task will be in the state TASK_UNINTERRUPTIBLE during the sleep.
*/
@@ -82,10 +82,10 @@ static inline void usleep_range(unsigned long min, unsigned long max)
* @min: Minimum time in microseconds to sleep
* @max: Maximum time in microseconds to sleep
*
* For basic information please refere to usleep_range_state().
* For basic information please refer to usleep_range_state().
*
* The sleeping task has the state TASK_IDLE during the sleep to prevent
* contribution to the load avarage.
* contribution to the load average.
*/
static inline void usleep_range_idle(unsigned long min, unsigned long max)
{
@@ -96,7 +96,7 @@ static inline void usleep_range_idle(unsigned long min, unsigned long max)
* ssleep - wrapper for seconds around msleep
* @seconds: Requested sleep duration in seconds
*
* Please refere to msleep() for detailed information.
* Please refer to msleep() for detailed information.
*/
static inline void ssleep(unsigned int seconds)
{
include/linux/find.h
+2
View File
@@ -44,6 +44,8 @@ unsigned long _find_next_bit_le(const unsigned long *addr, unsigned
long size, unsigned long offset);
#endif
unsigned long find_random_bit(const unsigned long *addr, unsigned long size);
#ifndef find_next_bit
/**
* find_next_bit - find the next set bit in a memory region
include/linux/hrtimer.h
+5 -9
View File
@@ -154,14 +154,11 @@ static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)
return ktime_to_ns(timer->node.expires);
}
ktime_t hrtimer_cb_get_time(const struct hrtimer *timer);
static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
{
return ktime_sub(timer->node.expires, timer->base->get_time());
}
static inline ktime_t hrtimer_cb_get_time(struct hrtimer *timer)
{
return timer->base->get_time();
return ktime_sub(timer->node.expires, hrtimer_cb_get_time(timer));
}
static inline int hrtimer_is_hres_active(struct hrtimer *timer)
@@ -200,8 +197,7 @@ __hrtimer_expires_remaining_adjusted(const struct hrtimer *timer, ktime_t now)
static inline ktime_t
hrtimer_expires_remaining_adjusted(const struct hrtimer *timer)
{
return __hrtimer_expires_remaining_adjusted(timer,
timer->base->get_time());
return __hrtimer_expires_remaining_adjusted(timer, hrtimer_cb_get_time(timer));
}
#ifdef CONFIG_TIMERFD
@@ -363,7 +359,7 @@ hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval);
static inline u64 hrtimer_forward_now(struct hrtimer *timer,
ktime_t interval)
{
return hrtimer_forward(timer, timer->base->get_time(), interval);
return hrtimer_forward(timer, hrtimer_cb_get_time(timer), interval);
}
/* Precise sleep: */
include/linux/hrtimer_defs.h
-2
View File
@@ -41,7 +41,6 @@
* @seq: seqcount around __run_hrtimer
* @running: pointer to the currently running hrtimer
* @active: red black tree root node for the active timers
* @get_time: function to retrieve the current time of the clock
* @offset: offset of this clock to the monotonic base
*/
struct hrtimer_clock_base {
@@ -51,7 +50,6 @@ struct hrtimer_clock_base {
seqcount_raw_spinlock_t seq;
struct hrtimer *running;
struct timerqueue_head active;
ktime_t (*get_time)(void);
ktime_t offset;
} __hrtimer_clock_base_align;
include/linux/iocontext.h
+3 -3
View File
@@ -118,8 +118,8 @@ struct task_struct;
#ifdef CONFIG_BLOCK
void put_io_context(struct io_context *ioc);
void exit_io_context(struct task_struct *task);
int __copy_io(unsigned long clone_flags, struct task_struct *tsk);
static inline int copy_io(unsigned long clone_flags, struct task_struct *tsk)
int __copy_io(u64 clone_flags, struct task_struct *tsk);
static inline int copy_io(u64 clone_flags, struct task_struct *tsk)
{
if (!current->io_context)
return 0;
@@ -129,7 +129,7 @@ static inline int copy_io(unsigned long clone_flags, struct task_struct *tsk)
struct io_context;
static inline void put_io_context(struct io_context *ioc) { }
static inline void exit_io_context(struct task_struct *task) { }
static inline int copy_io(unsigned long clone_flags, struct task_struct *tsk)
static inline int copy_io(u64 clone_flags, struct task_struct *tsk)
{
return 0;
}
include/linux/ipc_namespace.h
+2 -2
View File
@@ -129,7 +129,7 @@ static inline int mq_init_ns(struct ipc_namespace *ns) { return 0; }
#endif
#if defined(CONFIG_IPC_NS)
extern struct ipc_namespace *copy_ipcs(unsigned long flags,
extern struct ipc_namespace *copy_ipcs(u64 flags,
struct user_namespace *user_ns, struct ipc_namespace *ns);
static inline struct ipc_namespace *get_ipc_ns(struct ipc_namespace *ns)
@@ -151,7 +151,7 @@ static inline struct ipc_namespace *get_ipc_ns_not_zero(struct ipc_namespace *ns
extern void put_ipc_ns(struct ipc_namespace *ns);
#else
static inline struct ipc_namespace *copy_ipcs(unsigned long flags,
static inline struct ipc_namespace *copy_ipcs(u64 flags,
struct user_namespace *user_ns, struct ipc_namespace *ns)
{
if (flags & CLONE_NEWIPC)
include/linux/jiffies.h
+1 -1
View File
@@ -61,7 +61,7 @@
extern void register_refined_jiffies(long clock_tick_rate);
/* TICK_USEC is the time between ticks in usec assuming SHIFTED_HZ */
/* TICK_USEC is the time between ticks in usec */
#define TICK_USEC ((USEC_PER_SEC + HZ/2) / HZ)
/* USER_TICK_USEC is the time between ticks in usec assuming fake USER_HZ */
include/linux/lsm_hook_defs.h
+1 -1
View File
@@ -208,7 +208,7 @@ LSM_HOOK(int, 0, file_open, struct file *file)
LSM_HOOK(int, 0, file_post_open, struct file *file, int mask)
LSM_HOOK(int, 0, file_truncate, struct file *file)
LSM_HOOK(int, 0, task_alloc, struct task_struct *task,
unsigned long clone_flags)
u64 clone_flags)
LSM_HOOK(void, LSM_RET_VOID, task_free, struct task_struct *task)
LSM_HOOK(int, 0, cred_alloc_blank, struct cred *cred, gfp_t gfp)
LSM_HOOK(void, LSM_RET_VOID, cred_free, struct cred *cred)
include/linux/mnt_namespace.h
+1 -1
View File
@@ -11,7 +11,7 @@ struct fs_struct;
struct user_namespace;
struct ns_common;
extern struct mnt_namespace *copy_mnt_ns(unsigned long, struct mnt_namespace *,
extern struct mnt_namespace *copy_mnt_ns(u64, struct mnt_namespace *,
struct user_namespace *, struct fs_struct *);
extern void put_mnt_ns(struct mnt_namespace *ns);
DEFINE_FREE(put_mnt_ns, struct mnt_namespace *, if (!IS_ERR_OR_NULL(_T)) put_mnt_ns(_T))
include/linux/nodemask.h
+2 -14
View File
@@ -511,21 +511,9 @@ static __always_inline int num_node_state(enum node_states state)
static __always_inline int node_random(const nodemask_t *maskp)
{
#if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
int w, bit;
int node = find_random_bit(maskp->bits, MAX_NUMNODES);
w = nodes_weight(*maskp);
switch (w) {
case 0:
bit = NUMA_NO_NODE;
break;
case 1:
bit = first_node(*maskp);
break;
default:
bit = find_nth_bit(maskp->bits, MAX_NUMNODES, get_random_u32_below(w));
break;
}
return bit;
return node < MAX_NUMNODES ? node : NUMA_NO_NODE;
#else
return 0;
#endif
include/linux/nsproxy.h
+1 -1
View File
@@ -103,7 +103,7 @@ static inline struct cred *nsset_cred(struct nsset *set)
*
*/
int copy_namespaces(unsigned long flags, struct task_struct *tsk);
int copy_namespaces(u64 flags, struct task_struct *tsk);
void exit_task_namespaces(struct task_struct *tsk);
void switch_task_namespaces(struct task_struct *tsk, struct nsproxy *new);
int exec_task_namespaces(void);
include/linux/pid_namespace.h
+2 -2
View File
@@ -72,7 +72,7 @@ static inline int pidns_memfd_noexec_scope(struct pid_namespace *ns)
}
#endif
extern struct pid_namespace *copy_pid_ns(unsigned long flags,
extern struct pid_namespace *copy_pid_ns(u64 flags,
struct user_namespace *user_ns, struct pid_namespace *ns);
extern void zap_pid_ns_processes(struct pid_namespace *pid_ns);
extern int reboot_pid_ns(struct pid_namespace *pid_ns, int cmd);
@@ -91,7 +91,7 @@ static inline int pidns_memfd_noexec_scope(struct pid_namespace *ns)
return 0;
}
static inline struct pid_namespace *copy_pid_ns(unsigned long flags,
static inline struct pid_namespace *copy_pid_ns(u64 flags,
struct user_namespace *user_ns, struct pid_namespace *ns)
{
if (flags & CLONE_NEWPID)
include/linux/posix-timers.h
+5
View File
@@ -37,6 +37,11 @@ static inline int clockid_to_fd(const clockid_t clk)
return ~(clk >> 3);
}
static inline bool clockid_aux_valid(clockid_t id)
{
return IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS) && id >= CLOCK_AUX && id <= CLOCK_AUX_LAST;
}
#ifdef CONFIG_POSIX_TIMERS
#include <linux/signal_types.h>
include/linux/rseq.h
+2 -2
View File
@@ -65,7 +65,7 @@ static inline void rseq_migrate(struct task_struct *t)
* If parent process has a registered restartable sequences area, the
* child inherits. Unregister rseq for a clone with CLONE_VM set.
*/
static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
static inline void rseq_fork(struct task_struct *t, u64 clone_flags)
{
if (clone_flags & CLONE_VM) {
t->rseq = NULL;
@@ -107,7 +107,7 @@ static inline void rseq_preempt(struct task_struct *t)
static inline void rseq_migrate(struct task_struct *t)
{
}
static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
static inline void rseq_fork(struct task_struct *t, u64 clone_flags)
{
}
static inline void rseq_execve(struct task_struct *t)
include/linux/sched/task.h
+1 -1
View File
@@ -63,7 +63,7 @@ extern int lockdep_tasklist_lock_is_held(void);
extern asmlinkage void schedule_tail(struct task_struct *prev);
extern void init_idle(struct task_struct *idle, int cpu);
extern int sched_fork(unsigned long clone_flags, struct task_struct *p);
extern int sched_fork(u64 clone_flags, struct task_struct *p);
extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs);
extern void sched_cancel_fork(struct task_struct *p);
extern void sched_post_fork(struct task_struct *p);
include/linux/security.h
+2 -2
View File
@@ -437,7 +437,7 @@ int security_file_receive(struct file *file);
int security_file_open(struct file *file);
int security_file_post_open(struct file *file, int mask);
int security_file_truncate(struct file *file);
int security_task_alloc(struct task_struct *task, unsigned long clone_flags);
int security_task_alloc(struct task_struct *task, u64 clone_flags);
void security_task_free(struct task_struct *task);
int security_cred_alloc_blank(struct cred *cred, gfp_t gfp);
void security_cred_free(struct cred *cred);
@@ -1140,7 +1140,7 @@ static inline int security_file_truncate(struct file *file)
}
static inline int security_task_alloc(struct task_struct *task,
unsigned long clone_flags)
u64 clone_flags)
{
return 0;
}
include/linux/sem.h
+2 -2
View File
@@ -9,12 +9,12 @@ struct task_struct;
#ifdef CONFIG_SYSVIPC
extern int copy_semundo(unsigned long clone_flags, struct task_struct *tsk);
extern int copy_semundo(u64 clone_flags, struct task_struct *tsk);
extern void exit_sem(struct task_struct *tsk);
#else
static inline int copy_semundo(unsigned long clone_flags, struct task_struct *tsk)
static inline int copy_semundo(u64 clone_flags, struct task_struct *tsk)
{
return 0;
}
include/linux/time_namespace.h
+2 -2
View File
@@ -43,7 +43,7 @@ static inline struct time_namespace *get_time_ns(struct time_namespace *ns)
return ns;
}
struct time_namespace *copy_time_ns(unsigned long flags,
struct time_namespace *copy_time_ns(u64 flags,
struct user_namespace *user_ns,
struct time_namespace *old_ns);
void free_time_ns(struct time_namespace *ns);
@@ -129,7 +129,7 @@ static inline void put_time_ns(struct time_namespace *ns)
}
static inline
struct time_namespace *copy_time_ns(unsigned long flags,
struct time_namespace *copy_time_ns(u64 flags,
struct user_namespace *user_ns,
struct time_namespace *old_ns)
{
include/linux/timekeeper_internal.h
+43 -3
View File
@@ -11,6 +11,22 @@
#include <linux/jiffies.h>
#include <linux/time.h>
/**
* timekeeper_ids - IDs for various time keepers in the kernel
* @TIMEKEEPER_CORE: The central core timekeeper managing system time
* @TIMEKEEPER_AUX_FIRST: The first AUX timekeeper
* @TIMEKEEPER_AUX_LAST: The last AUX timekeeper
* @TIMEKEEPERS_MAX: The maximum number of timekeepers managed
*/
enum timekeeper_ids {
TIMEKEEPER_CORE,
#ifdef CONFIG_POSIX_AUX_CLOCKS
TIMEKEEPER_AUX_FIRST,
TIMEKEEPER_AUX_LAST = TIMEKEEPER_AUX_FIRST + MAX_AUX_CLOCKS - 1,
#endif
TIMEKEEPERS_MAX,
};
/**
* struct tk_read_base - base structure for timekeeping readout
* @clock: Current clocksource used for timekeeping.
@@ -51,12 +67,16 @@ struct tk_read_base {
* @offs_real: Offset clock monotonic -> clock realtime
* @offs_boot: Offset clock monotonic -> clock boottime
* @offs_tai: Offset clock monotonic -> clock tai
* @offs_aux: Offset clock monotonic -> clock AUX
* @coarse_nsec: The nanoseconds part for coarse time getters
* @id: The timekeeper ID
* @tkr_raw: The readout base structure for CLOCK_MONOTONIC_RAW
* @raw_sec: CLOCK_MONOTONIC_RAW time in seconds
* @clock_was_set_seq: The sequence number of clock was set events
* @cs_was_changed_seq: The sequence number of clocksource change events
* @clock_valid: Indicator for valid clock
* @monotonic_to_boot: CLOCK_MONOTONIC to CLOCK_BOOTTIME offset
* @monotonic_to_aux: CLOCK_MONOTONIC to CLOCK_AUX offset
* @cycle_interval: Number of clock cycles in one NTP interval
* @xtime_interval: Number of clock shifted nano seconds in one NTP
* interval.
@@ -95,13 +115,19 @@ struct tk_read_base {
* @monotonic_to_boottime is a timespec64 representation of @offs_boot to
* accelerate the VDSO update for CLOCK_BOOTTIME.
*
* @offs_aux is used by the auxiliary timekeepers which do not utilize any
* of the regular timekeeper offset fields.
*
* @monotonic_to_aux is a timespec64 representation of @offs_aux to
* accelerate the VDSO update for CLOCK_AUX.
*
* The cacheline ordering of the structure is optimized for in kernel usage of
* the ktime_get() and ktime_get_ts64() family of time accessors. Struct
* timekeeper is prepended in the core timekeeping code with a sequence count,
* which results in the following cacheline layout:
*
* 0: seqcount, tkr_mono
* 1: xtime_sec ... coarse_nsec
* 1: xtime_sec ... id
* 2: tkr_raw, raw_sec
* 3,4: Internal variables
*
@@ -121,8 +147,12 @@ struct timekeeper {
struct timespec64 wall_to_monotonic;
ktime_t offs_real;
ktime_t offs_boot;
ktime_t offs_tai;
union {
ktime_t offs_tai;
ktime_t offs_aux;
};
u32 coarse_nsec;
enum timekeeper_ids id;
/* Cacheline 2: */
struct tk_read_base tkr_raw;
@@ -131,8 +161,12 @@ struct timekeeper {
/* Cachline 3 and 4 (timekeeping internal variables): */
unsigned int clock_was_set_seq;
u8 cs_was_changed_seq;
u8 clock_valid;
struct timespec64 monotonic_to_boot;
union {
struct timespec64 monotonic_to_boot;
struct timespec64 monotonic_to_aux;
};
u64 cycle_interval;
u64 xtime_interval;
@@ -163,4 +197,10 @@ static inline void update_vsyscall_tz(void)
}
#endif
#if defined(CONFIG_GENERIC_GETTIMEOFDAY) && defined(CONFIG_POSIX_AUX_CLOCKS)
extern void vdso_time_update_aux(struct timekeeper *tk);
#else
static inline void vdso_time_update_aux(struct timekeeper *tk) { }
#endif
#endif /* _LINUX_TIMEKEEPER_INTERNAL_H */
include/linux/timekeeping.h
+12
View File
@@ -44,6 +44,7 @@ extern void ktime_get_ts64(struct timespec64 *ts);
extern void ktime_get_real_ts64(struct timespec64 *tv);
extern void ktime_get_coarse_ts64(struct timespec64 *ts);
extern void ktime_get_coarse_real_ts64(struct timespec64 *ts);
extern void ktime_get_clock_ts64(clockid_t id, struct timespec64 *ts);
/* Multigrain timestamp interfaces */
extern void ktime_get_coarse_real_ts64_mg(struct timespec64 *ts);
@@ -263,6 +264,17 @@ extern bool timekeeping_rtc_skipresume(void);
extern void timekeeping_inject_sleeptime64(const struct timespec64 *delta);
/*
* Auxiliary clock interfaces
*/
#ifdef CONFIG_POSIX_AUX_CLOCKS
extern bool ktime_get_aux(clockid_t id, ktime_t *kt);
extern bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *kt);
#else
static inline bool ktime_get_aux(clockid_t id, ktime_t *kt) { return false; }
static inline bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *kt) { return false; }
#endif
/**
* struct system_time_snapshot - simultaneous raw/real time capture with
* counter value
include/linux/uprobes.h
+2 -2
View File
@@ -142,7 +142,7 @@ extern void uprobe_start_dup_mmap(void);
extern void uprobe_end_dup_mmap(void);
extern void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm);
extern void uprobe_free_utask(struct task_struct *t);
extern void uprobe_copy_process(struct task_struct *t, unsigned long flags);
extern void uprobe_copy_process(struct task_struct *t, u64 flags);
extern int uprobe_post_sstep_notifier(struct pt_regs *regs);
extern int uprobe_pre_sstep_notifier(struct pt_regs *regs);
extern void uprobe_notify_resume(struct pt_regs *regs);
@@ -218,7 +218,7 @@ static inline bool uprobe_deny_signal(void)
static inline void uprobe_free_utask(struct task_struct *t)
{
}
static inline void uprobe_copy_process(struct task_struct *t, unsigned long flags)
static inline void uprobe_copy_process(struct task_struct *t, u64 flags)
{
}
static inline void uprobe_clear_state(struct mm_struct *mm)
include/linux/user_events.h
+2 -2
View File
@@ -33,7 +33,7 @@ extern void user_event_mm_dup(struct task_struct *t,
extern void user_event_mm_remove(struct task_struct *t);
static inline void user_events_fork(struct task_struct *t,
unsigned long clone_flags)
u64 clone_flags)
{
struct user_event_mm *old_mm;
@@ -68,7 +68,7 @@ static inline void user_events_exit(struct task_struct *t)
}
#else
static inline void user_events_fork(struct task_struct *t,
unsigned long clone_flags)
u64 clone_flags)
{
}
include/linux/utsname.h
+2 -2
View File
@@ -35,7 +35,7 @@ static inline void get_uts_ns(struct uts_namespace *ns)
refcount_inc(&ns->ns.count);
}
extern struct uts_namespace *copy_utsname(unsigned long flags,
extern struct uts_namespace *copy_utsname(u64 flags,
struct user_namespace *user_ns, struct uts_namespace *old_ns);
extern void free_uts_ns(struct uts_namespace *ns);
@@ -55,7 +55,7 @@ static inline void put_uts_ns(struct uts_namespace *ns)
{
}
static inline struct uts_namespace *copy_utsname(unsigned long flags,
static inline struct uts_namespace *copy_utsname(u64 flags,
struct user_namespace *user_ns, struct uts_namespace *old_ns)
{
if (flags & CLONE_NEWUTS)
include/net/net_namespace.h
+2 -2
View File
@@ -204,7 +204,7 @@ struct net {
extern struct net init_net;
#ifdef CONFIG_NET_NS
struct net *copy_net_ns(unsigned long flags, struct user_namespace *user_ns,
struct net *copy_net_ns(u64 flags, struct user_namespace *user_ns,
struct net *old_net);
void net_ns_get_ownership(const struct net *net, kuid_t *uid, kgid_t *gid);
@@ -216,7 +216,7 @@ struct net *get_net_ns_by_fd(int fd);
#else /* CONFIG_NET_NS */
#include <linux/sched.h>
#include <linux/nsproxy.h>
static inline struct net *copy_net_ns(unsigned long flags,
static inline struct net *copy_net_ns(u64 flags,
struct user_namespace *user_ns, struct net *old_net)
{
if (flags & CLONE_NEWNET)
include/trace/events/task.h
+3 -3
View File
@@ -8,14 +8,14 @@
TRACE_EVENT(task_newtask,
TP_PROTO(struct task_struct *task, unsigned long clone_flags),
TP_PROTO(struct task_struct *task, u64 clone_flags),
TP_ARGS(task, clone_flags),
TP_STRUCT__entry(
__field( pid_t, pid)
__array( char, comm, TASK_COMM_LEN)
__field( unsigned long, clone_flags)
__field( u64, clone_flags)
__field( short, oom_score_adj)
),
@@ -26,7 +26,7 @@ TRACE_EVENT(task_newtask,
__entry->oom_score_adj = task->signal->oom_score_adj;
),
TP_printk("pid=%d comm=%s clone_flags=%lx oom_score_adj=%hd",
TP_printk("pid=%d comm=%s clone_flags=%llx oom_score_adj=%hd",
__entry->pid, __entry->comm,
__entry->clone_flags, __entry->oom_score_adj)
);
include/uapi/linux/time.h
+11
View File
@@ -64,6 +64,17 @@ struct timezone {
#define CLOCK_TAI 11
#define MAX_CLOCKS 16
/*
* AUX clock support. AUXiliary clocks are dynamically configured by
* enabling a clock ID. These clock can be steered independently of the
* core timekeeper. The kernel can support up to 8 auxiliary clocks, but
* the actual limit depends on eventual architecture constraints vs. VDSO.
*/
#define CLOCK_AUX MAX_CLOCKS
#define MAX_AUX_CLOCKS 8
#define CLOCK_AUX_LAST (CLOCK_AUX + MAX_AUX_CLOCKS - 1)
#define CLOCKS_MASK (CLOCK_REALTIME | CLOCK_MONOTONIC)
#define CLOCKS_MONO CLOCK_MONOTONIC
include/vdso/auxclock.h
+13
View File
@@ -0,0 +1,13 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _VDSO_AUXCLOCK_H
#define _VDSO_AUXCLOCK_H
#include <uapi/linux/time.h>
#include <uapi/linux/types.h>
static __always_inline u64 aux_clock_resolution_ns(void)
{
return 1;
}
#endif /* _VDSO_AUXCLOCK_H */
include/vdso/datapage.h
+3
View File
@@ -38,6 +38,7 @@ struct vdso_arch_data {
#endif
#define VDSO_BASES (CLOCK_TAI + 1)
#define VDSO_BASE_AUX 0
#define VDSO_HRES (BIT(CLOCK_REALTIME) | \
BIT(CLOCK_MONOTONIC) | \
BIT(CLOCK_BOOTTIME) | \
@@ -117,6 +118,7 @@ struct vdso_clock {
* @arch_data: architecture specific data (optional, defaults
* to an empty struct)
* @clock_data: clocksource related data (array)
* @aux_clock_data: auxiliary clocksource related data (array)
* @tz_minuteswest: minutes west of Greenwich
* @tz_dsttime: type of DST correction
* @hrtimer_res: hrtimer resolution
@@ -133,6 +135,7 @@ struct vdso_time_data {
struct arch_vdso_time_data arch_data;
struct vdso_clock clock_data[CS_BASES];
struct vdso_clock aux_clock_data[MAX_AUX_CLOCKS];
s32 tz_minuteswest;
s32 tz_dsttime;
include/vdso/jiffies.h
+1 -1
View File
@@ -5,7 +5,7 @@
#include <asm/param.h> /* for HZ */
#include <vdso/time64.h>
/* TICK_NSEC is the time between ticks in nsec assuming SHIFTED_HZ */
/* TICK_NSEC is the time between ticks in nsec */
#define TICK_NSEC ((NSEC_PER_SEC+HZ/2)/HZ)
#endif /* __VDSO_JIFFIES_H */
ipc/namespace.c
+1 -1
View File
@@ -106,7 +106,7 @@ fail:
return ERR_PTR(err);
}
struct ipc_namespace *copy_ipcs(unsigned long flags,
struct ipc_namespace *copy_ipcs(u64 flags,
struct user_namespace *user_ns, struct ipc_namespace *ns)
{
if (!(flags & CLONE_NEWIPC))
ipc/sem.c
+1 -1
View File
@@ -2303,7 +2303,7 @@ SYSCALL_DEFINE3(semop, int, semid, struct sembuf __user *, tsops,
* parent and child tasks.
*/
int copy_semundo(unsigned long clone_flags, struct task_struct *tsk)
int copy_semundo(u64 clone_flags, struct task_struct *tsk)
{
struct sem_undo_list *undo_list;
int error;
kernel/cgroup/namespace.c
+1 -1
View File
@@ -47,7 +47,7 @@ void free_cgroup_ns(struct cgroup_namespace *ns)
}
EXPORT_SYMBOL(free_cgroup_ns);
struct cgroup_namespace *copy_cgroup_ns(unsigned long flags,
struct cgroup_namespace *copy_cgroup_ns(u64 flags,
struct user_namespace *user_ns,
struct cgroup_namespace *old_ns)
{
kernel/cred.c
+1 -1
View File
@@ -287,7 +287,7 @@ struct cred *prepare_exec_creds(void)
* The new process gets the current process's subjective credentials as its
* objective and subjective credentials
*/
int copy_creds(struct task_struct *p, unsigned long clone_flags)
int copy_creds(struct task_struct *p, u64 clone_flags)
{
struct cred *new;
int ret;
kernel/events/uprobes.c
+1 -1
View File
@@ -1873,7 +1873,7 @@ static void dup_xol_work(struct callback_head *work)
/*
* Called in context of a new clone/fork from copy_process.
*/
void uprobe_copy_process(struct task_struct *t, unsigned long flags)
void uprobe_copy_process(struct task_struct *t, u64 flags)
{
struct uprobe_task *utask = current->utask;
struct mm_struct *mm = current->mm;
kernel/fork.c
+4 -4
View File
@@ -1702,7 +1702,7 @@ fail_nomem:
return NULL;
}
static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
static int copy_mm(u64 clone_flags, struct task_struct *tsk)
{
struct mm_struct *mm, *oldmm;
@@ -1740,7 +1740,7 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
return 0;
}
static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
static int copy_fs(u64 clone_flags, struct task_struct *tsk)
{
struct fs_struct *fs = current->fs;
if (clone_flags & CLONE_FS) {
@@ -1761,7 +1761,7 @@ static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
return 0;
}
static int copy_files(unsigned long clone_flags, struct task_struct *tsk,
static int copy_files(u64 clone_flags, struct task_struct *tsk,
int no_files)
{
struct files_struct *oldf, *newf;
@@ -1840,7 +1840,7 @@ static void posix_cpu_timers_init_group(struct signal_struct *sig)
posix_cputimers_group_init(pct, cpu_limit);
}
static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
static int copy_signal(u64 clone_flags, struct task_struct *tsk)
{
struct signal_struct *sig;
kernel/nsproxy.c
+2 -2
View File
@@ -64,7 +64,7 @@ static inline struct nsproxy *create_nsproxy(void)
* Return the newly created nsproxy. Do not attach this to the task,
* leave it to the caller to do proper locking and attach it to task.
*/
static struct nsproxy *create_new_namespaces(unsigned long flags,
static struct nsproxy *create_new_namespaces(u64 flags,
struct task_struct *tsk, struct user_namespace *user_ns,
struct fs_struct *new_fs)
{
@@ -148,7 +148,7 @@ out_ns:
* called from clone. This now handles copy for nsproxy and all
* namespaces therein.
*/
int copy_namespaces(unsigned long flags, struct task_struct *tsk)
int copy_namespaces(u64 flags, struct task_struct *tsk)
{
struct nsproxy *old_ns = tsk->nsproxy;
struct user_namespace *user_ns = task_cred_xxx(tsk, user_ns);
kernel/pid_namespace.c
+1 -1
View File
@@ -143,7 +143,7 @@ static void destroy_pid_namespace(struct pid_namespace *ns)
call_rcu(&ns->rcu, delayed_free_pidns);
}
struct pid_namespace *copy_pid_ns(unsigned long flags,
struct pid_namespace *copy_pid_ns(u64 flags,
struct user_namespace *user_ns, struct pid_namespace *old_ns)
{
if (!(flags & CLONE_NEWPID))
kernel/sched/core.c
+3 -3
View File
@@ -894,7 +894,7 @@ void hrtick_start(struct rq *rq, u64 delay)
* doesn't make sense and can cause timer DoS.
*/
delta = max_t(s64, delay, 10000LL);
rq->hrtick_time = ktime_add_ns(timer->base->get_time(), delta);
rq->hrtick_time = ktime_add_ns(hrtimer_cb_get_time(timer), delta);
if (rq == this_rq())
__hrtick_restart(rq);
@@ -4407,7 +4407,7 @@ int wake_up_state(struct task_struct *p, unsigned int state)
* __sched_fork() is basic setup which is also used by sched_init() to
* initialize the boot CPU's idle task.
*/
static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
static void __sched_fork(u64 clone_flags, struct task_struct *p)
{
p->on_rq = 0;
@@ -4645,7 +4645,7 @@ late_initcall(sched_core_sysctl_init);
/*
* fork()/clone()-time setup:
*/
int sched_fork(unsigned long clone_flags, struct task_struct *p)
int sched_fork(u64 clone_flags, struct task_struct *p)
{
__sched_fork(clone_flags, p);
/*
kernel/sched/fair.c
+1 -1
View File
@@ -3537,7 +3537,7 @@ out:
}
}
void init_numa_balancing(unsigned long clone_flags, struct task_struct *p)
void init_numa_balancing(u64 clone_flags, struct task_struct *p)
{
int mm_users = 0;
struct mm_struct *mm = p->mm;
kernel/sched/sched.h
+2 -2
View File
@@ -1963,12 +1963,12 @@ extern void sched_setnuma(struct task_struct *p, int node);
extern int migrate_task_to(struct task_struct *p, int cpu);
extern int migrate_swap(struct task_struct *p, struct task_struct *t,
int cpu, int scpu);
extern void init_numa_balancing(unsigned long clone_flags, struct task_struct *p);
extern void init_numa_balancing(u64 clone_flags, struct task_struct *p);
#else /* !CONFIG_NUMA_BALANCING: */
static inline void
init_numa_balancing(unsigned long clone_flags, struct task_struct *p)
init_numa_balancing(u64 clone_flags, struct task_struct *p)
{
}
kernel/time/Kconfig
+13 -2
View File
@@ -82,9 +82,9 @@ config CONTEXT_TRACKING_IDLE
help
Tracks idle state on behalf of RCU.
if GENERIC_CLOCKEVENTS
menu "Timers subsystem"
if GENERIC_CLOCKEVENTS
# Core internal switch. Selected by NO_HZ_COMMON / HIGH_RES_TIMERS. This is
# only related to the tick functionality. Oneshot clockevent devices
# are supported independent of this.
@@ -208,6 +208,17 @@ config CLOCKSOURCE_WATCHDOG_MAX_SKEW_US
interval and NTP's maximum frequency drift of 500 parts
per million. If the clocksource is good enough for NTP,
it is good enough for the clocksource watchdog!
endif
config POSIX_AUX_CLOCKS
bool "Enable auxiliary POSIX clocks"
depends on POSIX_TIMERS
help
Auxiliary POSIX clocks are clocks which can be steered
independently of the core timekeeper, which controls the
MONOTONIC, REALTIME, BOOTTIME and TAI clocks. They are useful to
provide e.g. lockless time accessors to independent PTP clocks
and other clock domains, which are not correlated to the TAI/NTP
notion of time.
endmenu
endif
kernel/time/Makefile
+1 -1
View File
@@ -20,7 +20,7 @@ obj-$(CONFIG_LEGACY_TIMER_TICK) += tick-legacy.o
ifeq ($(CONFIG_SMP),y)
obj-$(CONFIG_NO_HZ_COMMON) += timer_migration.o
endif
obj-$(CONFIG_HAVE_GENERIC_VDSO) += vsyscall.o
obj-$(CONFIG_GENERIC_GETTIMEOFDAY) += vsyscall.o
obj-$(CONFIG_DEBUG_FS) += timekeeping_debug.o
obj-$(CONFIG_TEST_UDELAY) += test_udelay.o
obj-$(CONFIG_TIME_NS) += namespace.o
kernel/time/alarmtimer.c
+1 -1
View File
@@ -35,7 +35,7 @@
/**
* struct alarm_base - Alarm timer bases
* @lock: Lock for syncrhonized access to the base
* @lock: Lock for synchronized access to the base
* @timerqueue: Timerqueue head managing the list of events
* @get_ktime: Function to read the time correlating to the base
* @get_timespec: Function to read the namespace time correlating to the base
kernel/time/clocksource.c
+7 -15
View File
@@ -144,7 +144,7 @@ static u64 suspend_start;
* Default for maximum permissible skew when cs->uncertainty_margin is
* not specified, and the lower bound even when cs->uncertainty_margin
* is specified. This is also the default that is used when registering
* clocks with unspecifed cs->uncertainty_margin, so this macro is used
* clocks with unspecified cs->uncertainty_margin, so this macro is used
* even in CONFIG_CLOCKSOURCE_WATCHDOG=n kernels.
*/
#define WATCHDOG_MAX_SKEW (MAX_SKEW_USEC * NSEC_PER_USEC)
@@ -252,7 +252,7 @@ enum wd_read_status {
static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow)
{
int64_t md = 2 * watchdog->uncertainty_margin;
int64_t md = watchdog->uncertainty_margin;
unsigned int nretries, max_retries;
int64_t wd_delay, wd_seq_delay;
u64 wd_end, wd_end2;
@@ -323,9 +323,7 @@ static void clocksource_verify_choose_cpus(void)
return;
/* Make sure to select at least one CPU other than the current CPU. */
cpu = cpumask_first(cpu_online_mask);
if (cpu == smp_processor_id())
cpu = cpumask_next(cpu, cpu_online_mask);
cpu = cpumask_any_but(cpu_online_mask, smp_processor_id());
if (WARN_ON_ONCE(cpu >= nr_cpu_ids))
return;
cpumask_set_cpu(cpu, &cpus_chosen);
@@ -342,10 +340,7 @@ static void clocksource_verify_choose_cpus(void)
* CPUs that are currently online.
*/
for (i = 1; i < n; i++) {
cpu = get_random_u32_below(nr_cpu_ids);
cpu = cpumask_next(cpu - 1, cpu_online_mask);
if (cpu >= nr_cpu_ids)
cpu = cpumask_first(cpu_online_mask);
cpu = cpumask_random(cpu_online_mask);
if (!WARN_ON_ONCE(cpu >= nr_cpu_ids))
cpumask_set_cpu(cpu, &cpus_chosen);
}
@@ -412,9 +407,8 @@ void clocksource_verify_percpu(struct clocksource *cs)
if (!cpumask_empty(&cpus_behind))
pr_warn(" CPUs %*pbl behind CPU %d for clocksource %s.\n",
cpumask_pr_args(&cpus_behind), testcpu, cs->name);
if (!cpumask_empty(&cpus_ahead) || !cpumask_empty(&cpus_behind))
pr_warn(" CPU %d check durations %lldns - %lldns for clocksource %s.\n",
testcpu, cs_nsec_min, cs_nsec_max, cs->name);
pr_info(" CPU %d check durations %lldns - %lldns for clocksource %s.\n",
testcpu, cs_nsec_min, cs_nsec_max, cs->name);
}
EXPORT_SYMBOL_GPL(clocksource_verify_percpu);
@@ -589,9 +583,7 @@ static void clocksource_watchdog(struct timer_list *unused)
* Cycle through CPUs to check if the CPUs stay synchronized
* to each other.
*/
next_cpu = cpumask_next(raw_smp_processor_id(), cpu_online_mask);
if (next_cpu >= nr_cpu_ids)
next_cpu = cpumask_first(cpu_online_mask);
next_cpu = cpumask_next_wrap(raw_smp_processor_id(), cpu_online_mask);
/*
* Arm timer if not already pending: could race with concurrent
kernel/time/hrtimer.c
+33 -22
View File
@@ -59,6 +59,7 @@
#define HRTIMER_ACTIVE_ALL (HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD)
static void retrigger_next_event(void *arg);
static ktime_t __hrtimer_cb_get_time(clockid_t clock_id);
/*
* The timer bases:
@@ -76,42 +77,34 @@ DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
{
.index = HRTIMER_BASE_MONOTONIC,
.clockid = CLOCK_MONOTONIC,
.get_time = &ktime_get,
},
{
.index = HRTIMER_BASE_REALTIME,
.clockid = CLOCK_REALTIME,
.get_time = &ktime_get_real,
},
{
.index = HRTIMER_BASE_BOOTTIME,
.clockid = CLOCK_BOOTTIME,
.get_time = &ktime_get_boottime,
},
{
.index = HRTIMER_BASE_TAI,
.clockid = CLOCK_TAI,
.get_time = &ktime_get_clocktai,
},
{
.index = HRTIMER_BASE_MONOTONIC_SOFT,
.clockid = CLOCK_MONOTONIC,
.get_time = &ktime_get,
},
{
.index = HRTIMER_BASE_REALTIME_SOFT,
.clockid = CLOCK_REALTIME,
.get_time = &ktime_get_real,
},
{
.index = HRTIMER_BASE_BOOTTIME_SOFT,
.clockid = CLOCK_BOOTTIME,
.get_time = &ktime_get_boottime,
},
{
.index = HRTIMER_BASE_TAI_SOFT,
.clockid = CLOCK_TAI,
.get_time = &ktime_get_clocktai,
},
},
.csd = CSD_INIT(retrigger_next_event, NULL)
@@ -208,7 +201,7 @@ static bool hrtimer_suitable_target(struct hrtimer *timer, struct hrtimer_clock_
/*
* The offline local CPU can't be the default target if the
* next remote target event is after this timer. Keep the
* elected new base. An IPI will we issued to reprogram
* elected new base. An IPI will be issued to reprogram
* it as a last resort.
*/
if (!hrtimer_base_is_online(this_cpu_base))
@@ -787,10 +780,10 @@ static void retrigger_next_event(void *arg)
* of the next expiring timer is enough. The return from the SMP
* function call will take care of the reprogramming in case the
* CPU was in a NOHZ idle sleep.
*
* In periodic low resolution mode, the next softirq expiration
* must also be updated.
*/
if (!hrtimer_hres_active(base) && !tick_nohz_active)
return;
raw_spin_lock(&base->lock);
hrtimer_update_base(base);
if (hrtimer_hres_active(base))
@@ -920,7 +913,7 @@ static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,
return true;
/* Extra check for softirq clock bases */
if (base->clockid < HRTIMER_BASE_MONOTONIC_SOFT)
if (base->index < HRTIMER_BASE_MONOTONIC_SOFT)
continue;
if (cpu_base->softirq_activated)
continue;
@@ -1253,7 +1246,7 @@ static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
remove_hrtimer(timer, base, true, force_local);
if (mode & HRTIMER_MODE_REL)
tim = ktime_add_safe(tim, base->get_time());
tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid));
tim = hrtimer_update_lowres(timer, tim, mode);
@@ -1574,10 +1567,10 @@ u64 hrtimer_next_event_without(const struct hrtimer *exclude)
static inline int hrtimer_clockid_to_base(clockid_t clock_id)
{
switch (clock_id) {
case CLOCK_REALTIME:
return HRTIMER_BASE_REALTIME;
case CLOCK_MONOTONIC:
return HRTIMER_BASE_MONOTONIC;
case CLOCK_REALTIME:
return HRTIMER_BASE_REALTIME;
case CLOCK_BOOTTIME:
return HRTIMER_BASE_BOOTTIME;
case CLOCK_TAI:
@@ -1588,6 +1581,29 @@ static inline int hrtimer_clockid_to_base(clockid_t clock_id)
}
}
static ktime_t __hrtimer_cb_get_time(clockid_t clock_id)
{
switch (clock_id) {
case CLOCK_MONOTONIC:
return ktime_get();
case CLOCK_REALTIME:
return ktime_get_real();
case CLOCK_BOOTTIME:
return ktime_get_boottime();
case CLOCK_TAI:
return ktime_get_clocktai();
default:
WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id);
return ktime_get();
}
}
ktime_t hrtimer_cb_get_time(const struct hrtimer *timer)
{
return __hrtimer_cb_get_time(timer->base->clockid);
}
EXPORT_SYMBOL_GPL(hrtimer_cb_get_time);
static void __hrtimer_setup(struct hrtimer *timer,
enum hrtimer_restart (*function)(struct hrtimer *),
clockid_t clock_id, enum hrtimer_mode mode)
@@ -1726,7 +1742,7 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
lockdep_assert_held(&cpu_base->lock);
debug_deactivate(timer);
debug_hrtimer_deactivate(timer);
base->running = timer;
/*
@@ -2295,11 +2311,6 @@ int hrtimers_cpu_dying(unsigned int dying_cpu)
&new_base->clock_base[i]);
}
/*
* The migration might have changed the first expiring softirq
* timer on this CPU. Update it.
*/
__hrtimer_get_next_event(new_base, HRTIMER_ACTIVE_SOFT);
/* Tell the other CPU to retrigger the next event */
smp_call_function_single(ncpu, retrigger_next_event, NULL, 0);
kernel/time/itimer.c
+1 -2
View File
@@ -163,8 +163,7 @@ void posixtimer_rearm_itimer(struct task_struct *tsk)
struct hrtimer *tmr = &tsk->signal->real_timer;
if (!hrtimer_is_queued(tmr) && tsk->signal->it_real_incr != 0) {
hrtimer_forward(tmr, tmr->base->get_time(),
tsk->signal->it_real_incr);
hrtimer_forward_now(tmr, tsk->signal->it_real_incr);
hrtimer_restart(tmr);
}
}
kernel/time/namespace.c
+6 -1
View File
@@ -130,7 +130,7 @@ fail:
*
* Return: timens_for_children namespace or ERR_PTR.
*/
struct time_namespace *copy_time_ns(unsigned long flags,
struct time_namespace *copy_time_ns(u64 flags,
struct user_namespace *user_ns, struct time_namespace *old_ns)
{
if (!(flags & CLONE_NEWTIME))
@@ -242,6 +242,11 @@ static void timens_set_vvar_page(struct task_struct *task,
for (i = 0; i < CS_BASES; i++)
timens_setup_vdso_clock_data(&vc[i], ns);
if (IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) {
for (i = 0; i < ARRAY_SIZE(vdata->aux_clock_data); i++)
timens_setup_vdso_clock_data(&vdata->aux_clock_data[i], ns);
}
out:
mutex_unlock(&offset_lock);
}
kernel/time/ntp.c
+40 -32
View File
@@ -18,6 +18,7 @@
#include <linux/module.h>
#include <linux/rtc.h>
#include <linux/audit.h>
#include <linux/timekeeper_internal.h>
#include "ntp_internal.h"
#include "timekeeping_internal.h"
@@ -86,14 +87,16 @@ struct ntp_data {
#endif
};
static struct ntp_data tk_ntp_data = {
.tick_usec = USER_TICK_USEC,
.time_state = TIME_OK,
.time_status = STA_UNSYNC,
.time_constant = 2,
.time_maxerror = NTP_PHASE_LIMIT,
.time_esterror = NTP_PHASE_LIMIT,
.ntp_next_leap_sec = TIME64_MAX,
static struct ntp_data tk_ntp_data[TIMEKEEPERS_MAX] = {
[ 0 ... TIMEKEEPERS_MAX - 1 ] = {
.tick_usec = USER_TICK_USEC,
.time_state = TIME_OK,
.time_status = STA_UNSYNC,
.time_constant = 2,
.time_maxerror = NTP_PHASE_LIMIT,
.time_esterror = NTP_PHASE_LIMIT,
.ntp_next_leap_sec = TIME64_MAX,
},
};
#define SECS_PER_DAY 86400
@@ -300,7 +303,7 @@ static void ntp_update_offset(struct ntp_data *ntpdata, long offset)
* Select how the frequency is to be controlled
* and in which mode (PLL or FLL).
*/
real_secs = __ktime_get_real_seconds();
real_secs = ktime_get_ntp_seconds(ntpdata - tk_ntp_data);
secs = (long)(real_secs - ntpdata->time_reftime);
if (unlikely(ntpdata->time_status & STA_FREQHOLD))
secs = 0;
@@ -348,33 +351,38 @@ static void __ntp_clear(struct ntp_data *ntpdata)
/**
* ntp_clear - Clears the NTP state variables
* @tkid: Timekeeper ID to be able to select proper ntp data array member
*/
void ntp_clear(void)
void ntp_clear(unsigned int tkid)
{
__ntp_clear(&tk_ntp_data);
__ntp_clear(&tk_ntp_data[tkid]);
}
u64 ntp_tick_length(void)
u64 ntp_tick_length(unsigned int tkid)
{
return tk_ntp_data.tick_length;
return tk_ntp_data[tkid].tick_length;
}
/**
* ntp_get_next_leap - Returns the next leapsecond in CLOCK_REALTIME ktime_t
* @tkid: Timekeeper ID
*
* Provides the time of the next leapsecond against CLOCK_REALTIME in
* a ktime_t format. Returns KTIME_MAX if no leapsecond is pending.
* Returns: For @tkid == TIMEKEEPER_CORE this provides the time of the next
* leap second against CLOCK_REALTIME in a ktime_t format if a
* leap second is pending. KTIME_MAX otherwise.
*/
ktime_t ntp_get_next_leap(void)
ktime_t ntp_get_next_leap(unsigned int tkid)
{
struct ntp_data *ntpdata = &tk_ntp_data;
ktime_t ret;
struct ntp_data *ntpdata = &tk_ntp_data[TIMEKEEPER_CORE];
if (tkid != TIMEKEEPER_CORE)
return KTIME_MAX;
if ((ntpdata->time_state == TIME_INS) && (ntpdata->time_status & STA_INS))
return ktime_set(ntpdata->ntp_next_leap_sec, 0);
ret = KTIME_MAX;
return ret;
return KTIME_MAX;
}
/*
@@ -387,9 +395,9 @@ ktime_t ntp_get_next_leap(void)
*
* Also handles leap second processing, and returns leap offset
*/
int second_overflow(time64_t secs)
int second_overflow(unsigned int tkid, time64_t secs)
{
struct ntp_data *ntpdata = &tk_ntp_data;
struct ntp_data *ntpdata = &tk_ntp_data[tkid];
s64 delta;
int leap = 0;
s32 rem;
@@ -605,7 +613,7 @@ static inline int update_rtc(struct timespec64 *to_set, unsigned long *offset_ns
*/
static inline bool ntp_synced(void)
{
return !(tk_ntp_data.time_status & STA_UNSYNC);
return !(tk_ntp_data[TIMEKEEPER_CORE].time_status & STA_UNSYNC);
}
/*
@@ -702,7 +710,7 @@ static inline void process_adj_status(struct ntp_data *ntpdata, const struct __k
* reference time to current time.
*/
if (!(ntpdata->time_status & STA_PLL) && (txc->status & STA_PLL))
ntpdata->time_reftime = __ktime_get_real_seconds();
ntpdata->time_reftime = ktime_get_ntp_seconds(ntpdata - tk_ntp_data);
/* only set allowed bits */
ntpdata->time_status &= STA_RONLY;
@@ -759,10 +767,10 @@ static inline void process_adjtimex_modes(struct ntp_data *ntpdata, const struct
* adjtimex() mainly allows reading (and writing, if superuser) of
* kernel time-keeping variables. used by xntpd.
*/
int __do_adjtimex(struct __kernel_timex *txc, const struct timespec64 *ts,
s32 *time_tai, struct audit_ntp_data *ad)
int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct timespec64 *ts,
s32 *time_tai, struct audit_ntp_data *ad)
{
struct ntp_data *ntpdata = &tk_ntp_data;
struct ntp_data *ntpdata = &tk_ntp_data[tkid];
int result;
if (txc->modes & ADJ_ADJTIME) {
@@ -1031,8 +1039,8 @@ static void hardpps_update_phase(struct ntp_data *ntpdata, long error)
*/
void __hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
{
struct ntp_data *ntpdata = &tk_ntp_data[TIMEKEEPER_CORE];
struct pps_normtime pts_norm, freq_norm;
struct ntp_data *ntpdata = &tk_ntp_data;
pts_norm = pps_normalize_ts(*phase_ts);
@@ -1083,18 +1091,18 @@ void __hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_t
static int __init ntp_tick_adj_setup(char *str)
{
int rc = kstrtos64(str, 0, &tk_ntp_data.ntp_tick_adj);
int rc = kstrtos64(str, 0, &tk_ntp_data[TIMEKEEPER_CORE].ntp_tick_adj);
if (rc)
return rc;
tk_ntp_data.ntp_tick_adj <<= NTP_SCALE_SHIFT;
tk_ntp_data[TIMEKEEPER_CORE].ntp_tick_adj <<= NTP_SCALE_SHIFT;
return 1;
}
__setup("ntp_tick_adj=", ntp_tick_adj_setup);
void __init ntp_init(void)
{
ntp_clear();
for (int id = 0; id < TIMEKEEPERS_MAX; id++)
__ntp_clear(tk_ntp_data + id);
ntp_init_cmos_sync();
}
kernel/time/ntp_internal.h
+6 -7
View File
@@ -3,14 +3,13 @@
#define _LINUX_NTP_INTERNAL_H
extern void ntp_init(void);
extern void ntp_clear(void);
extern void ntp_clear(unsigned int tkid);
/* Returns how long ticks are at present, in ns / 2^NTP_SCALE_SHIFT. */
extern u64 ntp_tick_length(void);
extern ktime_t ntp_get_next_leap(void);
extern int second_overflow(time64_t secs);
extern int __do_adjtimex(struct __kernel_timex *txc,
const struct timespec64 *ts,
s32 *time_tai, struct audit_ntp_data *ad);
extern u64 ntp_tick_length(unsigned int tkid);
extern ktime_t ntp_get_next_leap(unsigned int tkid);
extern int second_overflow(unsigned int tkid, time64_t secs);
extern int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct timespec64 *ts,
s32 *time_tai, struct audit_ntp_data *ad);
extern void __hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts);
#if defined(CONFIG_GENERIC_CMOS_UPDATE) || defined(CONFIG_RTC_SYSTOHC)
kernel/time/posix-timers.c
+7 -5
View File
@@ -299,8 +299,7 @@ static void common_hrtimer_rearm(struct k_itimer *timr)
{
struct hrtimer *timer = &timr->it.real.timer;
timr->it_overrun += hrtimer_forward(timer, timer->base->get_time(),
timr->it_interval);
timr->it_overrun += hrtimer_forward_now(timer, timr->it_interval);
hrtimer_restart(timer);
}
@@ -535,7 +534,7 @@ static int do_timer_create(clockid_t which_clock, struct sigevent *event,
goto out;
}
/*
* After succesful copy out, the timer ID is visible to user space
* After successful copy out, the timer ID is visible to user space
* now but not yet valid because new_timer::signal low order bit is 1.
*
* Complete the initialization with the clock specific create
@@ -825,7 +824,7 @@ static void common_hrtimer_arm(struct k_itimer *timr, ktime_t expires,
hrtimer_setup(&timr->it.real.timer, posix_timer_fn, timr->it_clock, mode);
if (!absolute)
expires = ktime_add_safe(expires, timer->base->get_time());
expires = ktime_add_safe(expires, hrtimer_cb_get_time(timer));
hrtimer_set_expires(timer, expires);
if (!sigev_none)
@@ -1243,7 +1242,7 @@ SYSCALL_DEFINE2(clock_adjtime, const clockid_t, which_clock,
* sys_clock_settime(). The kernel internal timekeeping is always using
* nanoseconds precision independent of the clocksource device which is
* used to read the time from. The resolution of that device only
* affects the presicion of the time returned by sys_clock_gettime().
* affects the precision of the time returned by sys_clock_gettime().
*
* Returns:
* 0 Success. @tp contains the resolution
@@ -1526,6 +1525,9 @@ static const struct k_clock * const posix_clocks[] = {
[CLOCK_REALTIME_ALARM] = &alarm_clock,
[CLOCK_BOOTTIME_ALARM] = &alarm_clock,
[CLOCK_TAI] = &clock_tai,
#ifdef CONFIG_POSIX_AUX_CLOCKS
[CLOCK_AUX ... CLOCK_AUX_LAST] = &clock_aux,
#endif
};
static const struct k_clock *clockid_to_kclock(const clockid_t id)
kernel/time/posix-timers.h
+1
View File
@@ -41,6 +41,7 @@ extern const struct k_clock clock_posix_dynamic;
extern const struct k_clock clock_process;
extern const struct k_clock clock_thread;
extern const struct k_clock alarm_clock;
extern const struct k_clock clock_aux;
void posix_timer_queue_signal(struct k_itimer *timr);
kernel/time/sched_clock.c
+3 -3
View File
@@ -174,8 +174,7 @@ static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt)
return HRTIMER_RESTART;
}
void __init
sched_clock_register(u64 (*read)(void), int bits, unsigned long rate)
void sched_clock_register(u64 (*read)(void), int bits, unsigned long rate)
{
u64 res, wrap, new_mask, new_epoch, cyc, ns;
u32 new_mult, new_shift;
@@ -216,7 +215,7 @@ sched_clock_register(u64 (*read)(void), int bits, unsigned long rate)
update_clock_read_data(&rd);
if (sched_clock_timer.function != NULL) {
if (ACCESS_PRIVATE(&sched_clock_timer, function) != NULL) {
/* update timeout for clock wrap */
hrtimer_start(&sched_clock_timer, cd.wrap_kt,
HRTIMER_MODE_REL_HARD);
@@ -247,6 +246,7 @@ sched_clock_register(u64 (*read)(void), int bits, unsigned long rate)
pr_debug("Registered %pS as sched_clock source\n", read);
}
EXPORT_SYMBOL_GPL(sched_clock_register);
void __init generic_sched_clock_init(void)
{
kernel/time/tick-sched.c
+5 -6
View File
@@ -1174,16 +1174,15 @@ static bool report_idle_softirq(void)
return false;
}
if (ratelimit >= 10)
return false;
/* On RT, softirq handling may be waiting on some lock */
if (local_bh_blocked())
return false;
pr_warn("NOHZ tick-stop error: local softirq work is pending, handler #%02x!!!\n",
pending);
ratelimit++;
if (ratelimit < 10) {
pr_warn("NOHZ tick-stop error: local softirq work is pending, handler #%02x!!!\n",
pending);
ratelimit++;
}
return true;
}
kernel/time/time.c
+1
View File
@@ -858,6 +858,7 @@ struct timespec64 timespec64_add_safe(const struct timespec64 lhs,
return res;
}
EXPORT_SYMBOL_GPL(timespec64_add_safe);
/**
* get_timespec64 - get user's time value into kernel space
kernel/time/timekeeping.c
+578 -86
View File
@@ -6,6 +6,7 @@
#include <linux/timekeeper_internal.h>
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/kobject.h>
#include <linux/percpu.h>
#include <linux/init.h>
#include <linux/mm.h>
@@ -25,6 +26,8 @@
#include <linux/audit.h>
#include <linux/random.h>
#include <vdso/auxclock.h>
#include "tick-internal.h"
#include "ntp_internal.h"
#include "timekeeping_internal.h"
@@ -53,7 +56,38 @@ struct tk_data {
raw_spinlock_t lock;
} ____cacheline_aligned;
static struct tk_data tk_core;
static struct tk_data timekeeper_data[TIMEKEEPERS_MAX];
/* The core timekeeper */
#define tk_core (timekeeper_data[TIMEKEEPER_CORE])
#ifdef CONFIG_POSIX_AUX_CLOCKS
static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts)
{
return ktime_get_aux_ts64(CLOCK_AUX + tkid - TIMEKEEPER_AUX_FIRST, ts);
}
static inline bool tk_is_aux(const struct timekeeper *tk)
{
return tk->id >= TIMEKEEPER_AUX_FIRST && tk->id <= TIMEKEEPER_AUX_LAST;
}
#else
static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts)
{
return false;
}
static inline bool tk_is_aux(const struct timekeeper *tk)
{
return false;
}
#endif
static inline void tk_update_aux_offs(struct timekeeper *tk, ktime_t offs)
{
tk->offs_aux = offs;
tk->monotonic_to_aux = ktime_to_timespec64(offs);
}
/* flag for if timekeeping is suspended */
int __read_mostly timekeeping_suspended;
@@ -113,6 +147,16 @@ static struct tk_fast tk_fast_raw ____cacheline_aligned = {
.base[1] = FAST_TK_INIT,
};
#ifdef CONFIG_POSIX_AUX_CLOCKS
static __init void tk_aux_setup(void);
static void tk_aux_update_clocksource(void);
static void tk_aux_advance(void);
#else
static inline void tk_aux_setup(void) { }
static inline void tk_aux_update_clocksource(void) { }
static inline void tk_aux_advance(void) { }
#endif
unsigned long timekeeper_lock_irqsave(void)
{
unsigned long flags;
@@ -601,7 +645,7 @@ EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
*/
static inline void tk_update_leap_state(struct timekeeper *tk)
{
tk->next_leap_ktime = ntp_get_next_leap();
tk->next_leap_ktime = ntp_get_next_leap(tk->id);
if (tk->next_leap_ktime != KTIME_MAX)
/* Convert to monotonic time */
tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);
@@ -663,7 +707,7 @@ static void timekeeping_restore_shadow(struct tk_data *tkd)
static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int action)
{
struct timekeeper *tk = &tk_core.shadow_timekeeper;
struct timekeeper *tk = &tkd->shadow_timekeeper;
lockdep_assert_held(&tkd->lock);
@@ -678,18 +722,22 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act
if (action & TK_CLEAR_NTP) {
tk->ntp_error = 0;
ntp_clear();
ntp_clear(tk->id);
}
tk_update_leap_state(tk);
tk_update_ktime_data(tk);
update_vsyscall(tk);
update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;
update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw);
if (tk->id == TIMEKEEPER_CORE) {
update_vsyscall(tk);
update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw);
} else if (tk_is_aux(tk)) {
vdso_time_update_aux(tk);
}
if (action & TK_CLOCK_WAS_SET)
tk->clock_was_set_seq++;
@@ -975,9 +1023,14 @@ time64_t ktime_get_real_seconds(void)
EXPORT_SYMBOL_GPL(ktime_get_real_seconds);
/**
* __ktime_get_real_seconds - The same as ktime_get_real_seconds
* but without the sequence counter protect. This internal function
* is called just when timekeeping lock is already held.
* __ktime_get_real_seconds - Unprotected access to CLOCK_REALTIME seconds
*
* The same as ktime_get_real_seconds() but without the sequence counter
* protection. This function is used in restricted contexts like the x86 MCE
* handler and in KGDB. It's unprotected on 32-bit vs. concurrent half
* completed modification and only to be used for such critical contexts.
*
* Returns: Racy snapshot of the CLOCK_REALTIME seconds value
*/
noinstr time64_t __ktime_get_real_seconds(void)
{
@@ -1412,41 +1465,73 @@ int do_settimeofday64(const struct timespec64 *ts)
}
EXPORT_SYMBOL(do_settimeofday64);
static inline bool timekeeper_is_core_tk(struct timekeeper *tk)
{
return !IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS) || tk->id == TIMEKEEPER_CORE;
}
/**
* timekeeping_inject_offset - Adds or subtracts from the current time.
* __timekeeping_inject_offset - Adds or subtracts from the current time.
* @tkd: Pointer to the timekeeper to modify
* @ts: Pointer to the timespec variable containing the offset
*
* Adds or subtracts an offset value from the current time.
*/
static int timekeeping_inject_offset(const struct timespec64 *ts)
static int __timekeeping_inject_offset(struct tk_data *tkd, const struct timespec64 *ts)
{
struct timekeeper *tks = &tkd->shadow_timekeeper;
struct timespec64 tmp;
if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC)
return -EINVAL;
scoped_guard (raw_spinlock_irqsave, &tk_core.lock) {
struct timekeeper *tks = &tk_core.shadow_timekeeper;
struct timespec64 tmp;
timekeeping_forward_now(tks);
timekeeping_forward_now(tks);
if (timekeeper_is_core_tk(tks)) {
/* Make sure the proposed value is valid */
tmp = timespec64_add(tk_xtime(tks), *ts);
if (timespec64_compare(&tks->wall_to_monotonic, ts) > 0 ||
!timespec64_valid_settod(&tmp)) {
timekeeping_restore_shadow(&tk_core);
timekeeping_restore_shadow(tkd);
return -EINVAL;
}
tk_xtime_add(tks, ts);
tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, *ts));
timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
} else {
struct tk_read_base *tkr_mono = &tks->tkr_mono;
ktime_t now, offs;
/* Get the current time */
now = ktime_add_ns(tkr_mono->base, timekeeping_get_ns(tkr_mono));
/* Add the relative offset change */
offs = ktime_add(tks->offs_aux, timespec64_to_ktime(*ts));
/* Prevent that the resulting time becomes negative */
if (ktime_add(now, offs) < 0) {
timekeeping_restore_shadow(tkd);
return -EINVAL;
}
tk_update_aux_offs(tks, offs);
}
/* Signal hrtimers about time change */
clock_was_set(CLOCK_SET_WALL);
timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL);
return 0;
}
static int timekeeping_inject_offset(const struct timespec64 *ts)
{
int ret;
scoped_guard (raw_spinlock_irqsave, &tk_core.lock)
ret = __timekeeping_inject_offset(&tk_core, ts);
/* Signal hrtimers about time change */
if (!ret)
clock_was_set(CLOCK_SET_WALL);
return ret;
}
/*
* Indicates if there is an offset between the system clock and the hardware
* clock/persistent clock/rtc.
@@ -1522,6 +1607,8 @@ static int change_clocksource(void *data)
timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
}
tk_aux_update_clocksource();
if (old) {
if (old->disable)
old->disable(old);
@@ -1573,6 +1660,39 @@ void ktime_get_raw_ts64(struct timespec64 *ts)
}
EXPORT_SYMBOL(ktime_get_raw_ts64);
/**
* ktime_get_clock_ts64 - Returns time of a clock in a timespec
* @id: POSIX clock ID of the clock to read
* @ts: Pointer to the timespec64 to be set
*
* The timestamp is invalidated (@ts->sec is set to -1) if the
* clock @id is not available.
*/
void ktime_get_clock_ts64(clockid_t id, struct timespec64 *ts)
{
/* Invalidate time stamp */
ts->tv_sec = -1;
ts->tv_nsec = 0;
switch (id) {
case CLOCK_REALTIME:
ktime_get_real_ts64(ts);
return;
case CLOCK_MONOTONIC:
ktime_get_ts64(ts);
return;
case CLOCK_MONOTONIC_RAW:
ktime_get_raw_ts64(ts);
return;
case CLOCK_AUX ... CLOCK_AUX_LAST:
if (IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS))
ktime_get_aux_ts64(id, ts);
return;
default:
WARN_ON_ONCE(1);
}
}
EXPORT_SYMBOL_GPL(ktime_get_clock_ts64);
/**
* timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
@@ -1649,10 +1769,12 @@ read_persistent_wall_and_boot_offset(struct timespec64 *wall_time,
*boot_offset = ns_to_timespec64(local_clock());
}
static __init void tkd_basic_setup(struct tk_data *tkd)
static __init void tkd_basic_setup(struct tk_data *tkd, enum timekeeper_ids tk_id, bool valid)
{
raw_spin_lock_init(&tkd->lock);
seqcount_raw_spinlock_init(&tkd->seq, &tkd->lock);
tkd->timekeeper.id = tkd->shadow_timekeeper.id = tk_id;
tkd->timekeeper.clock_valid = tkd->shadow_timekeeper.clock_valid = valid;
}
/*
@@ -1682,7 +1804,8 @@ void __init timekeeping_init(void)
struct timekeeper *tks = &tk_core.shadow_timekeeper;
struct clocksource *clock;
tkd_basic_setup(&tk_core);
tkd_basic_setup(&tk_core, TIMEKEEPER_CORE, true);
tk_aux_setup();
read_persistent_wall_and_boot_offset(&wall_time, &boot_offset);
if (timespec64_valid_settod(&wall_time) &&
@@ -2034,7 +2157,7 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
*/
static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
{
u64 ntp_tl = ntp_tick_length();
u64 ntp_tl = ntp_tick_length(tk->id);
u32 mult;
/*
@@ -2115,7 +2238,7 @@ static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
}
/* Figure out if its a leap sec and apply if needed */
leap = second_overflow(tk->xtime_sec);
leap = second_overflow(tk->id, tk->xtime_sec);
if (unlikely(leap)) {
struct timespec64 ts;
@@ -2181,16 +2304,14 @@ static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
* timekeeping_advance - Updates the timekeeper to the current time and
* current NTP tick length
*/
static bool timekeeping_advance(enum timekeeping_adv_mode mode)
static bool __timekeeping_advance(struct tk_data *tkd, enum timekeeping_adv_mode mode)
{
struct timekeeper *tk = &tk_core.shadow_timekeeper;
struct timekeeper *real_tk = &tk_core.timekeeper;
struct timekeeper *tk = &tkd->shadow_timekeeper;
struct timekeeper *real_tk = &tkd->timekeeper;
unsigned int clock_set = 0;
int shift = 0, maxshift;
u64 offset, orig_offset;
guard(raw_spinlock_irqsave)(&tk_core.lock);
/* Make sure we're fully resumed: */
if (unlikely(timekeeping_suspended))
return false;
@@ -2214,7 +2335,7 @@ static bool timekeeping_advance(enum timekeeping_adv_mode mode)
shift = ilog2(offset) - ilog2(tk->cycle_interval);
shift = max(0, shift);
/* Bound shift to one less than what overflows tick_length */
maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
maxshift = (64 - (ilog2(ntp_tick_length(tk->id)) + 1)) - 1;
shift = min(shift, maxshift);
while (offset >= tk->cycle_interval) {
offset = logarithmic_accumulation(tk, offset, shift, &clock_set);
@@ -2239,19 +2360,27 @@ static bool timekeeping_advance(enum timekeeping_adv_mode mode)
if (orig_offset != offset)
tk_update_coarse_nsecs(tk);
timekeeping_update_from_shadow(&tk_core, clock_set);
timekeeping_update_from_shadow(tkd, clock_set);
return !!clock_set;
}
static bool timekeeping_advance(enum timekeeping_adv_mode mode)
{
guard(raw_spinlock_irqsave)(&tk_core.lock);
return __timekeeping_advance(&tk_core, mode);
}
/**
* update_wall_time - Uses the current clocksource to increment the wall time
*
* It also updates the enabled auxiliary clock timekeepers
*/
void update_wall_time(void)
{
if (timekeeping_advance(TK_ADV_TICK))
clock_was_set_delayed();
tk_aux_advance();
}
/**
@@ -2449,7 +2578,7 @@ ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
/*
* timekeeping_validate_timex - Ensures the timex is ok for use in do_adjtimex
*/
static int timekeeping_validate_timex(const struct __kernel_timex *txc)
static int timekeeping_validate_timex(const struct __kernel_timex *txc, bool aux_clock)
{
if (txc->modes & ADJ_ADJTIME) {
/* singleshot must not be used with any other mode bits */
@@ -2508,6 +2637,22 @@ static int timekeeping_validate_timex(const struct __kernel_timex *txc)
return -EINVAL;
}
if (aux_clock) {
/* Auxiliary clocks are similar to TAI and do not have leap seconds */
if (txc->modes & ADJ_STATUS &&
txc->status & (STA_INS | STA_DEL))
return -EINVAL;
/* No TAI offset setting */
if (txc->modes & ADJ_TAI)
return -EINVAL;
/* No PPS support either */
if (txc->modes & ADJ_STATUS &&
txc->status & (STA_PPSFREQ | STA_PPSTIME))
return -EINVAL;
}
return 0;
}
@@ -2526,74 +2671,103 @@ unsigned long random_get_entropy_fallback(void)
}
EXPORT_SYMBOL_GPL(random_get_entropy_fallback);
struct adjtimex_result {
struct audit_ntp_data ad;
struct timespec64 delta;
bool clock_set;
};
static int __do_adjtimex(struct tk_data *tkd, struct __kernel_timex *txc,
struct adjtimex_result *result)
{
struct timekeeper *tks = &tkd->shadow_timekeeper;
bool aux_clock = !timekeeper_is_core_tk(tks);
struct timespec64 ts;
s32 orig_tai, tai;
int ret;
/* Validate the data before disabling interrupts */
ret = timekeeping_validate_timex(txc, aux_clock);
if (ret)
return ret;
add_device_randomness(txc, sizeof(*txc));
if (!aux_clock)
ktime_get_real_ts64(&ts);
else
tk_get_aux_ts64(tkd->timekeeper.id, &ts);
add_device_randomness(&ts, sizeof(ts));
guard(raw_spinlock_irqsave)(&tkd->lock);
if (!tks->clock_valid)
return -ENODEV;
if (txc->modes & ADJ_SETOFFSET) {
result->delta.tv_sec = txc->time.tv_sec;
result->delta.tv_nsec = txc->time.tv_usec;
if (!(txc->modes & ADJ_NANO))
result->delta.tv_nsec *= 1000;
ret = __timekeeping_inject_offset(tkd, &result->delta);
if (ret)
return ret;
result->clock_set = true;
}
orig_tai = tai = tks->tai_offset;
ret = ntp_adjtimex(tks->id, txc, &ts, &tai, &result->ad);
if (tai != orig_tai) {
__timekeeping_set_tai_offset(tks, tai);
timekeeping_update_from_shadow(tkd, TK_CLOCK_WAS_SET);
result->clock_set = true;
} else {
tk_update_leap_state_all(tkd);
}
/* Update the multiplier immediately if frequency was set directly */
if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK))
result->clock_set |= __timekeeping_advance(tkd, TK_ADV_FREQ);
return ret;
}
/**
* do_adjtimex() - Accessor function to NTP __do_adjtimex function
* @txc: Pointer to kernel_timex structure containing NTP parameters
*/
int do_adjtimex(struct __kernel_timex *txc)
{
struct audit_ntp_data ad;
bool offset_set = false;
bool clock_set = false;
struct timespec64 ts;
struct adjtimex_result result = { };
int ret;
/* Validate the data before disabling interrupts */
ret = timekeeping_validate_timex(txc);
if (ret)
ret = __do_adjtimex(&tk_core, txc, &result);
if (ret < 0)
return ret;
add_device_randomness(txc, sizeof(*txc));
if (txc->modes & ADJ_SETOFFSET) {
struct timespec64 delta;
if (txc->modes & ADJ_SETOFFSET)
audit_tk_injoffset(result.delta);
delta.tv_sec = txc->time.tv_sec;
delta.tv_nsec = txc->time.tv_usec;
if (!(txc->modes & ADJ_NANO))
delta.tv_nsec *= 1000;
ret = timekeeping_inject_offset(&delta);
if (ret)
return ret;
audit_ntp_log(&result.ad);
offset_set = delta.tv_sec != 0;
audit_tk_injoffset(delta);
}
audit_ntp_init(&ad);
ktime_get_real_ts64(&ts);
add_device_randomness(&ts, sizeof(ts));
scoped_guard (raw_spinlock_irqsave, &tk_core.lock) {
struct timekeeper *tks = &tk_core.shadow_timekeeper;
s32 orig_tai, tai;
orig_tai = tai = tks->tai_offset;
ret = __do_adjtimex(txc, &ts, &tai, &ad);
if (tai != orig_tai) {
__timekeeping_set_tai_offset(tks, tai);
timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET);
clock_set = true;
} else {
tk_update_leap_state_all(&tk_core);
}
}
audit_ntp_log(&ad);
/* Update the multiplier immediately if frequency was set directly */
if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK))
clock_set |= timekeeping_advance(TK_ADV_FREQ);
if (clock_set)
if (result.clock_set)
clock_was_set(CLOCK_SET_WALL);
ntp_notify_cmos_timer(offset_set);
ntp_notify_cmos_timer(result.delta.tv_sec != 0);
return ret;
}
/*
* Invoked from NTP with the time keeper lock held, so lockless access is
* fine.
*/
long ktime_get_ntp_seconds(unsigned int id)
{
return timekeeper_data[id].timekeeper.xtime_sec;
}
#ifdef CONFIG_NTP_PPS
/**
* hardpps() - Accessor function to NTP __hardpps function
@@ -2607,3 +2781,321 @@ void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
}
EXPORT_SYMBOL(hardpps);
#endif /* CONFIG_NTP_PPS */
#ifdef CONFIG_POSIX_AUX_CLOCKS
#include "posix-timers.h"
/*
* Bitmap for the activated auxiliary timekeepers to allow lockless quick
* checks in the hot paths without touching extra cache lines. If set, then
* the state of the corresponding timekeeper has to be re-checked under
* timekeeper::lock.
*/
static unsigned long aux_timekeepers;
static inline unsigned int clockid_to_tkid(unsigned int id)
{
return TIMEKEEPER_AUX_FIRST + id - CLOCK_AUX;
}
static inline struct tk_data *aux_get_tk_data(clockid_t id)
{
if (!clockid_aux_valid(id))
return NULL;
return &timekeeper_data[clockid_to_tkid(id)];
}
/* Invoked from timekeeping after a clocksource change */
static void tk_aux_update_clocksource(void)
{
unsigned long active = READ_ONCE(aux_timekeepers);
unsigned int id;
for_each_set_bit(id, &active, BITS_PER_LONG) {
struct tk_data *tkd = &timekeeper_data[id + TIMEKEEPER_AUX_FIRST];
struct timekeeper *tks = &tkd->shadow_timekeeper;
guard(raw_spinlock_irqsave)(&tkd->lock);
if (!tks->clock_valid)
continue;
timekeeping_forward_now(tks);
tk_setup_internals(tks, tk_core.timekeeper.tkr_mono.clock);
timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL);
}
}
static void tk_aux_advance(void)
{
unsigned long active = READ_ONCE(aux_timekeepers);
unsigned int id;
/* Lockless quick check to avoid extra cache lines */
for_each_set_bit(id, &active, BITS_PER_LONG) {
struct tk_data *aux_tkd = &timekeeper_data[id + TIMEKEEPER_AUX_FIRST];
guard(raw_spinlock)(&aux_tkd->lock);
if (aux_tkd->shadow_timekeeper.clock_valid)
__timekeeping_advance(aux_tkd, TK_ADV_TICK);
}
}
/**
* ktime_get_aux - Get time for a AUX clock
* @id: ID of the clock to read (CLOCK_AUX...)
* @kt: Pointer to ktime_t to store the time stamp
*
* Returns: True if the timestamp is valid, false otherwise
*/
bool ktime_get_aux(clockid_t id, ktime_t *kt)
{
struct tk_data *aux_tkd = aux_get_tk_data(id);
struct timekeeper *aux_tk;
unsigned int seq;
ktime_t base;
u64 nsecs;
WARN_ON(timekeeping_suspended);
if (!aux_tkd)
return false;
aux_tk = &aux_tkd->timekeeper;
do {
seq = read_seqcount_begin(&aux_tkd->seq);
if (!aux_tk->clock_valid)
return false;
base = ktime_add(aux_tk->tkr_mono.base, aux_tk->offs_aux);
nsecs = timekeeping_get_ns(&aux_tk->tkr_mono);
} while (read_seqcount_retry(&aux_tkd->seq, seq));
*kt = ktime_add_ns(base, nsecs);
return true;
}
EXPORT_SYMBOL_GPL(ktime_get_aux);
/**
* ktime_get_aux_ts64 - Get time for a AUX clock
* @id: ID of the clock to read (CLOCK_AUX...)
* @ts: Pointer to timespec64 to store the time stamp
*
* Returns: True if the timestamp is valid, false otherwise
*/
bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *ts)
{
ktime_t now;
if (!ktime_get_aux(id, &now))
return false;
*ts = ktime_to_timespec64(now);
return true;
}
EXPORT_SYMBOL_GPL(ktime_get_aux_ts64);
static int aux_get_res(clockid_t id, struct timespec64 *tp)
{
if (!clockid_aux_valid(id))
return -ENODEV;
tp->tv_sec = aux_clock_resolution_ns() / NSEC_PER_SEC;
tp->tv_nsec = aux_clock_resolution_ns() % NSEC_PER_SEC;
return 0;
}
static int aux_get_timespec(clockid_t id, struct timespec64 *tp)
{
return ktime_get_aux_ts64(id, tp) ? 0 : -ENODEV;
}
static int aux_clock_set(const clockid_t id, const struct timespec64 *tnew)
{
struct tk_data *aux_tkd = aux_get_tk_data(id);
struct timekeeper *aux_tks;
ktime_t tnow, nsecs;
if (!timespec64_valid_settod(tnew))
return -EINVAL;
if (!aux_tkd)
return -ENODEV;
aux_tks = &aux_tkd->shadow_timekeeper;
guard(raw_spinlock_irq)(&aux_tkd->lock);
if (!aux_tks->clock_valid)
return -ENODEV;
/* Forward the timekeeper base time */
timekeeping_forward_now(aux_tks);
/*
* Get the updated base time. tkr_mono.base has not been
* updated yet, so do that first. That makes the update
* in timekeeping_update_from_shadow() redundant, but
* that's harmless. After that @tnow can be calculated
* by using tkr_mono::cycle_last, which has been set
* by timekeeping_forward_now().
*/
tk_update_ktime_data(aux_tks);
nsecs = timekeeping_cycles_to_ns(&aux_tks->tkr_mono, aux_tks->tkr_mono.cycle_last);
tnow = ktime_add(aux_tks->tkr_mono.base, nsecs);
/*
* Calculate the new AUX offset as delta to @tnow ("monotonic").
* That avoids all the tk::xtime back and forth conversions as
* xtime ("realtime") is not applicable for auxiliary clocks and
* kept in sync with "monotonic".
*/
tk_update_aux_offs(aux_tks, ktime_sub(timespec64_to_ktime(*tnew), tnow));
timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL);
return 0;
}
static int aux_clock_adj(const clockid_t id, struct __kernel_timex *txc)
{
struct tk_data *aux_tkd = aux_get_tk_data(id);
struct adjtimex_result result = { };
if (!aux_tkd)
return -ENODEV;
/*
* @result is ignored for now as there are neither hrtimers nor a
* RTC related to auxiliary clocks for now.
*/
return __do_adjtimex(aux_tkd, txc, &result);
}
const struct k_clock clock_aux = {
.clock_getres = aux_get_res,
.clock_get_timespec = aux_get_timespec,
.clock_set = aux_clock_set,
.clock_adj = aux_clock_adj,
};
static void aux_clock_enable(clockid_t id)
{
struct tk_read_base *tkr_raw = &tk_core.timekeeper.tkr_raw;
struct tk_data *aux_tkd = aux_get_tk_data(id);
struct timekeeper *aux_tks = &aux_tkd->shadow_timekeeper;
/* Prevent the core timekeeper from changing. */
guard(raw_spinlock_irq)(&tk_core.lock);
/*
* Setup the auxiliary clock assuming that the raw core timekeeper
* clock frequency conversion is close enough. Userspace has to
* adjust for the deviation via clock_adjtime(2).
*/
guard(raw_spinlock_nested)(&aux_tkd->lock);
/* Remove leftovers of a previous registration */
memset(aux_tks, 0, sizeof(*aux_tks));
/* Restore the timekeeper id */
aux_tks->id = aux_tkd->timekeeper.id;
/* Setup the timekeeper based on the current system clocksource */
tk_setup_internals(aux_tks, tkr_raw->clock);
/* Mark it valid and set it live */
aux_tks->clock_valid = true;
timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL);
}
static void aux_clock_disable(clockid_t id)
{
struct tk_data *aux_tkd = aux_get_tk_data(id);
guard(raw_spinlock_irq)(&aux_tkd->lock);
aux_tkd->shadow_timekeeper.clock_valid = false;
timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL);
}
static DEFINE_MUTEX(aux_clock_mutex);
static ssize_t aux_clock_enable_store(struct kobject *kobj, struct kobj_attribute *attr,
const char *buf, size_t count)
{
/* Lazy atoi() as name is "0..7" */
int id = kobj->name[0] & 0x7;
bool enable;
if (!capable(CAP_SYS_TIME))
return -EPERM;
if (kstrtobool(buf, &enable) < 0)
return -EINVAL;
guard(mutex)(&aux_clock_mutex);
if (enable == test_bit(id, &aux_timekeepers))
return count;
if (enable) {
aux_clock_enable(CLOCK_AUX + id);
set_bit(id, &aux_timekeepers);
} else {
aux_clock_disable(CLOCK_AUX + id);
clear_bit(id, &aux_timekeepers);
}
return count;
}
static ssize_t aux_clock_enable_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
{
unsigned long active = READ_ONCE(aux_timekeepers);
/* Lazy atoi() as name is "0..7" */
int id = kobj->name[0] & 0x7;
return sysfs_emit(buf, "%d\n", test_bit(id, &active));
}
static struct kobj_attribute aux_clock_enable_attr = __ATTR_RW(aux_clock_enable);
static struct attribute *aux_clock_enable_attrs[] = {
&aux_clock_enable_attr.attr,
NULL
};
static const struct attribute_group aux_clock_enable_attr_group = {
.attrs = aux_clock_enable_attrs,
};
static int __init tk_aux_sysfs_init(void)
{
struct kobject *auxo, *tko = kobject_create_and_add("time", kernel_kobj);
int ret = -ENOMEM;
if (!tko)
return ret;
auxo = kobject_create_and_add("aux_clocks", tko);
if (!auxo)
goto err_clean;
for (int i = 0; i < MAX_AUX_CLOCKS; i++) {
char id[2] = { [0] = '0' + i, };
struct kobject *clk = kobject_create_and_add(id, auxo);
if (!clk) {
ret = -ENOMEM;
goto err_clean;
}
ret = sysfs_create_group(clk, &aux_clock_enable_attr_group);
if (ret)
goto err_clean;
}
return 0;
err_clean:
kobject_put(auxo);
kobject_put(tko);
return ret;
}
late_initcall(tk_aux_sysfs_init);
static __init void tk_aux_setup(void)
{
for (int i = TIMEKEEPER_AUX_FIRST; i <= TIMEKEEPER_AUX_LAST; i++)
tkd_basic_setup(&timekeeper_data[i], i, false);
}
#endif /* CONFIG_POSIX_AUX_CLOCKS */
kernel/time/timekeeping_internal.h
+3
View File
@@ -45,4 +45,7 @@ static inline u64 clocksource_delta(u64 now, u64 last, u64 mask, u64 max_delta)
unsigned long timekeeper_lock_irqsave(void);
void timekeeper_unlock_irqrestore(unsigned long flags);
/* NTP specific interface to access the current seconds value */
long ktime_get_ntp_seconds(unsigned int id);
#endif /* _TIMEKEEPING_INTERNAL_H */
kernel/time/timer.c
+4 -3
View File
@@ -1458,10 +1458,11 @@ static int __try_to_del_timer_sync(struct timer_list *timer, bool shutdown)
base = lock_timer_base(timer, &flags);
if (base->running_timer != timer)
if (base->running_timer != timer) {
ret = detach_if_pending(timer, base, true);
if (shutdown)
timer->function = NULL;
if (shutdown)
timer->function = NULL;
}
raw_spin_unlock_irqrestore(&base->lock, flags);
kernel/time/timer_list.c
-2
View File
@@ -102,8 +102,6 @@ print_base(struct seq_file *m, struct hrtimer_clock_base *base, u64 now)
SEQ_printf(m, " .index: %d\n", base->index);
SEQ_printf(m, " .resolution: %u nsecs\n", hrtimer_resolution);
SEQ_printf(m, " .get_time: %ps\n", base->get_time);
#ifdef CONFIG_HIGH_RES_TIMERS
SEQ_printf(m, " .offset: %Lu nsecs\n",
(unsigned long long) ktime_to_ns(base->offset));
kernel/time/timer_migration.c
+161 -157
View File
@@ -420,6 +420,8 @@ static struct list_head *tmigr_level_list __read_mostly;
static unsigned int tmigr_hierarchy_levels __read_mostly;
static unsigned int tmigr_crossnode_level __read_mostly;
static struct tmigr_group *tmigr_root;
static DEFINE_PER_CPU(struct tmigr_cpu, tmigr_cpu);
#define TMIGR_NONE 0xFF
@@ -502,11 +504,6 @@ static bool tmigr_check_lonely(struct tmigr_group *group)
* @now: timer base monotonic
* @check: is set if there is the need to handle remote timers;
* required in tmigr_requires_handle_remote() only
* @tmc_active: this flag indicates, whether the CPU which triggers
* the hierarchy walk is !idle in the timer migration
* hierarchy. When the CPU is idle and the whole hierarchy is
* idle, only the first event of the top level has to be
* considered.
*/
struct tmigr_walk {
u64 nextexp;
@@ -517,16 +514,13 @@ struct tmigr_walk {
unsigned long basej;
u64 now;
bool check;
bool tmc_active;
};
typedef bool (*up_f)(struct tmigr_group *, struct tmigr_group *, struct tmigr_walk *);
static void __walk_groups(up_f up, struct tmigr_walk *data,
struct tmigr_cpu *tmc)
static void __walk_groups_from(up_f up, struct tmigr_walk *data,
struct tmigr_group *child, struct tmigr_group *group)
{
struct tmigr_group *child = NULL, *group = tmc->tmgroup;
do {
WARN_ON_ONCE(group->level >= tmigr_hierarchy_levels);
@@ -544,6 +538,12 @@ static void __walk_groups(up_f up, struct tmigr_walk *data,
} while (group);
}
static void __walk_groups(up_f up, struct tmigr_walk *data,
struct tmigr_cpu *tmc)
{
__walk_groups_from(up, data, NULL, tmc->tmgroup);
}
static void walk_groups(up_f up, struct tmigr_walk *data, struct tmigr_cpu *tmc)
{
lockdep_assert_held(&tmc->lock);
@@ -708,7 +708,7 @@ void tmigr_cpu_activate(void)
/*
* Returns true, if there is nothing to be propagated to the next level
*
* @data->firstexp is set to expiry of first gobal event of the (top level of
* @data->firstexp is set to expiry of first global event of the (top level of
* the) hierarchy, but only when hierarchy is completely idle.
*
* The child and group states need to be read under the lock, to prevent a race
@@ -1113,15 +1113,6 @@ static bool tmigr_requires_handle_remote_up(struct tmigr_group *group,
*/
if (!tmigr_check_migrator(group, childmask))
return true;
/*
* When there is a parent group and the CPU which triggered the
* hierarchy walk is not active, proceed the walk to reach the top level
* group before reading the next_expiry value.
*/
if (group->parent && !data->tmc_active)
return false;
/*
* The lock is required on 32bit architectures to read the variable
* consistently with a concurrent writer. On 64bit the lock is not
@@ -1166,7 +1157,6 @@ bool tmigr_requires_handle_remote(void)
data.now = get_jiffies_update(&jif);
data.childmask = tmc->groupmask;
data.firstexp = KTIME_MAX;
data.tmc_active = !tmc->idle;
data.check = false;
/*
@@ -1405,23 +1395,20 @@ u64 tmigr_quick_check(u64 nextevt)
return KTIME_MAX;
do {
if (!tmigr_check_lonely(group)) {
if (!tmigr_check_lonely(group))
return KTIME_MAX;
} else {
/*
* Since current CPU is active, events may not be sorted
* from bottom to the top because the CPU's event is ignored
* up to the top and its sibling's events not propagated upwards.
* Thus keep track of the lowest observed expiry.
*/
nextevt = min_t(u64, nextevt, READ_ONCE(group->next_expiry));
if (!group->parent)
return nextevt;
}
/*
* Since current CPU is active, events may not be sorted
* from bottom to the top because the CPU's event is ignored
* up to the top and its sibling's events not propagated upwards.
* Thus keep track of the lowest observed expiry.
*/
nextevt = min_t(u64, nextevt, READ_ONCE(group->next_expiry));
group = group->parent;
} while (group);
return KTIME_MAX;
return nextevt;
}
/*
@@ -1501,21 +1488,6 @@ static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl,
s.seq = 0;
atomic_set(&group->migr_state, s.state);
/*
* If this is a new top-level, prepare its groupmask in advance.
* This avoids accidents where yet another new top-level is
* created in the future and made visible before the current groupmask.
*/
if (list_empty(&tmigr_level_list[lvl])) {
group->groupmask = BIT(0);
/*
* The previous top level has prepared its groupmask already,
* simply account it as the first child.
*/
if (lvl > 0)
group->num_children = 1;
}
timerqueue_init_head(&group->events);
timerqueue_init(&group->groupevt.nextevt);
group->groupevt.nextevt.expires = KTIME_MAX;
@@ -1523,8 +1495,7 @@ static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl,
group->groupevt.ignore = true;
}
static struct tmigr_group *tmigr_get_group(unsigned int cpu, int node,
unsigned int lvl)
static struct tmigr_group *tmigr_get_group(int node, unsigned int lvl)
{
struct tmigr_group *tmp, *group = NULL;
@@ -1570,25 +1541,51 @@ static struct tmigr_group *tmigr_get_group(unsigned int cpu, int node,
return group;
}
static bool tmigr_init_root(struct tmigr_group *group, bool activate)
{
if (!group->parent && group != tmigr_root) {
/*
* This is the new top-level, prepare its groupmask in advance
* to avoid accidents where yet another new top-level is
* created in the future and made visible before this groupmask.
*/
group->groupmask = BIT(0);
WARN_ON_ONCE(activate);
return true;
}
return false;
}
static void tmigr_connect_child_parent(struct tmigr_group *child,
struct tmigr_group *parent,
bool activate)
{
struct tmigr_walk data;
raw_spin_lock_irq(&child->lock);
raw_spin_lock_nested(&parent->lock, SINGLE_DEPTH_NESTING);
if (activate) {
if (tmigr_init_root(parent, activate)) {
/*
* @child is the old top and @parent the new one. In this
* case groupmask is pre-initialized and @child already
* accounted, along with its new sibling corresponding to the
* CPU going up.
* The previous top level had prepared its groupmask already,
* simply account it in advance as the first child. If some groups
* have been created between the old and new root due to node
* mismatch, the new root's child will be intialized accordingly.
*/
WARN_ON_ONCE(child->groupmask != BIT(0) || parent->num_children != 2);
parent->num_children = 1;
}
/* Connecting old root to new root ? */
if (!parent->parent && activate) {
/*
* @child is the old top, or in case of node mismatch, some
* intermediate group between the old top and the new one in
* @parent. In this case the @child must be pre-accounted above
* as the first child. Its new inactive sibling corresponding
* to the CPU going up has been accounted as the second child.
*/
WARN_ON_ONCE(parent->num_children != 2);
child->groupmask = BIT(0);
} else {
/* Adding @child for the CPU going up to @parent. */
/* Common case adding @child for the CPU going up to @parent. */
child->groupmask = BIT(parent->num_children++);
}
@@ -1599,87 +1596,61 @@ static void tmigr_connect_child_parent(struct tmigr_group *child,
*/
smp_store_release(&child->parent, parent);
raw_spin_unlock(&parent->lock);
raw_spin_unlock_irq(&child->lock);
trace_tmigr_connect_child_parent(child);
if (!activate)
return;
/*
* To prevent inconsistent states, active children need to be active in
* the new parent as well. Inactive children are already marked inactive
* in the parent group:
*
* * When new groups were created by tmigr_setup_groups() starting from
* the lowest level (and not higher then one level below the current
* top level), then they are not active. They will be set active when
* the new online CPU comes active.
*
* * But if a new group above the current top level is required, it is
* mandatory to propagate the active state of the already existing
* child to the new parent. So tmigr_connect_child_parent() is
* executed with the formerly top level group (child) and the newly
* created group (parent).
*
* * It is ensured that the child is active, as this setup path is
* executed in hotplug prepare callback. This is exectued by an
* already connected and !idle CPU. Even if all other CPUs go idle,
* the CPU executing the setup will be responsible up to current top
* level group. And the next time it goes inactive, it will release
* the new childmask and parent to subsequent walkers through this
* @child. Therefore propagate active state unconditionally.
*/
data.childmask = child->groupmask;
/*
* There is only one new level per time (which is protected by
* tmigr_mutex). When connecting the child and the parent and set the
* child active when the parent is inactive, the parent needs to be the
* uppermost level. Otherwise there went something wrong!
*/
WARN_ON(!tmigr_active_up(parent, child, &data) && parent->parent);
}
static int tmigr_setup_groups(unsigned int cpu, unsigned int node)
static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
struct tmigr_group *start, bool activate)
{
struct tmigr_group *group, *child, **stack;
int top = 0, err = 0, i = 0;
struct list_head *lvllist;
int i, top = 0, err = 0, start_lvl = 0;
bool root_mismatch = false;
stack = kcalloc(tmigr_hierarchy_levels, sizeof(*stack), GFP_KERNEL);
if (!stack)
return -ENOMEM;
do {
group = tmigr_get_group(cpu, node, i);
if (start) {
stack[start->level] = start;
start_lvl = start->level + 1;
}
if (tmigr_root)
root_mismatch = tmigr_root->numa_node != node;
for (i = start_lvl; i < tmigr_hierarchy_levels; i++) {
group = tmigr_get_group(node, i);
if (IS_ERR(group)) {
err = PTR_ERR(group);
i--;
break;
}
top = i;
stack[i++] = group;
stack[i] = group;
/*
* When booting only less CPUs of a system than CPUs are
* available, not all calculated hierarchy levels are required.
* available, not all calculated hierarchy levels are required,
* unless a node mismatch is detected.
*
* The loop is aborted as soon as the highest level, which might
* be different from tmigr_hierarchy_levels, contains only a
* single group.
* single group, unless the nodes mismatch below tmigr_crossnode_level
*/
if (group->parent || list_is_singular(&tmigr_level_list[i - 1]))
if (group->parent)
break;
if ((!root_mismatch || i >= tmigr_crossnode_level) &&
list_is_singular(&tmigr_level_list[i]))
break;
}
} while (i < tmigr_hierarchy_levels);
/* Assert single root without parent */
if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels))
return -EINVAL;
/* Assert single root */
WARN_ON_ONCE(!err && !group->parent && !list_is_singular(&tmigr_level_list[top]));
while (i > 0) {
group = stack[--i];
for (; i >= start_lvl; i--) {
group = stack[i];
if (err < 0) {
list_del(&group->list);
@@ -1695,12 +1666,10 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node)
if (i == 0) {
struct tmigr_cpu *tmc = per_cpu_ptr(&tmigr_cpu, cpu);
raw_spin_lock_irq(&group->lock);
tmc->tmgroup = group;
tmc->groupmask = BIT(group->num_children++);
raw_spin_unlock_irq(&group->lock);
tmigr_init_root(group, activate);
trace_tmigr_connect_cpu_parent(tmc);
@@ -1708,42 +1677,58 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node)
continue;
} else {
child = stack[i - 1];
/* Will be activated at online time */
tmigr_connect_child_parent(child, group, false);
}
/* check if uppermost level was newly created */
if (top != i)
continue;
WARN_ON_ONCE(top == 0);
lvllist = &tmigr_level_list[top];
/*
* Newly created root level should have accounted the upcoming
* CPU's child group and pre-accounted the old root.
*/
if (group->num_children == 2 && list_is_singular(lvllist)) {
/*
* The target CPU must never do the prepare work, except
* on early boot when the boot CPU is the target. Otherwise
* it may spuriously activate the old top level group inside
* the new one (nevertheless whether old top level group is
* active or not) and/or release an uninitialized childmask.
*/
WARN_ON_ONCE(cpu == raw_smp_processor_id());
lvllist = &tmigr_level_list[top - 1];
list_for_each_entry(child, lvllist, list) {
if (child->parent)
continue;
tmigr_connect_child_parent(child, group, true);
}
tmigr_connect_child_parent(child, group, activate);
}
}
if (err < 0)
goto out;
if (activate) {
struct tmigr_walk data;
union tmigr_state state;
/*
* To prevent inconsistent states, active children need to be active in
* the new parent as well. Inactive children are already marked inactive
* in the parent group:
*
* * When new groups were created by tmigr_setup_groups() starting from
* the lowest level, then they are not active. They will be set active
* when the new online CPU comes active.
*
* * But if new groups above the current top level are required, it is
* mandatory to propagate the active state of the already existing
* child to the new parents. So tmigr_active_up() activates the
* new parents while walking up from the old root to the new.
*
* * It is ensured that @start is active, as this setup path is
* executed in hotplug prepare callback. This is executed by an
* already connected and !idle CPU. Even if all other CPUs go idle,
* the CPU executing the setup will be responsible up to current top
* level group. And the next time it goes inactive, it will release
* the new childmask and parent to subsequent walkers through this
* @child. Therefore propagate active state unconditionally.
*/
state.state = atomic_read(&start->migr_state);
WARN_ON_ONCE(!state.active);
WARN_ON_ONCE(!start->parent);
data.childmask = start->groupmask;
__walk_groups_from(tmigr_active_up, &data, start, start->parent);
}
/* Root update */
if (list_is_singular(&tmigr_level_list[top])) {
group = list_first_entry(&tmigr_level_list[top],
typeof(*group), list);
WARN_ON_ONCE(group->parent);
if (tmigr_root) {
/* Old root should be the same or below */
WARN_ON_ONCE(tmigr_root->level > top);
}
tmigr_root = group;
}
out:
kfree(stack);
return err;
@@ -1751,12 +1736,31 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node)
static int tmigr_add_cpu(unsigned int cpu)
{
struct tmigr_group *old_root = tmigr_root;
int node = cpu_to_node(cpu);
int ret;
mutex_lock(&tmigr_mutex);
ret = tmigr_setup_groups(cpu, node);
mutex_unlock(&tmigr_mutex);
guard(mutex)(&tmigr_mutex);
ret = tmigr_setup_groups(cpu, node, NULL, false);
/* Root has changed? Connect the old one to the new */
if (ret >= 0 && old_root && old_root != tmigr_root) {
/*
* The target CPU must never do the prepare work, except
* on early boot when the boot CPU is the target. Otherwise
* it may spuriously activate the old top level group inside
* the new one (nevertheless whether old top level group is
* active or not) and/or release an uninitialized childmask.
*/
WARN_ON_ONCE(cpu == raw_smp_processor_id());
/*
* The (likely) current CPU is expected to be online in the hierarchy,
* otherwise the old root may not be active as expected.
*/
WARN_ON_ONCE(!per_cpu_ptr(&tmigr_cpu, raw_smp_processor_id())->online);
ret = tmigr_setup_groups(-1, old_root->numa_node, old_root, true);
}
return ret;
}
kernel/time/vsyscall.c
+55 -15
View File
@@ -15,26 +15,25 @@
#include "timekeeping_internal.h"
static inline void fill_clock_configuration(struct vdso_clock *vc, const struct tk_read_base *base)
{
vc->cycle_last = base->cycle_last;
#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
vc->max_cycles = base->clock->max_cycles;
#endif
vc->mask = base->mask;
vc->mult = base->mult;
vc->shift = base->shift;
}
static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct timekeeper *tk)
{
struct vdso_clock *vc = vdata->clock_data;
struct vdso_timestamp *vdso_ts;
u64 nsec, sec;
vc[CS_HRES_COARSE].cycle_last = tk->tkr_mono.cycle_last;
#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
vc[CS_HRES_COARSE].max_cycles = tk->tkr_mono.clock->max_cycles;
#endif
vc[CS_HRES_COARSE].mask = tk->tkr_mono.mask;
vc[CS_HRES_COARSE].mult = tk->tkr_mono.mult;
vc[CS_HRES_COARSE].shift = tk->tkr_mono.shift;
vc[CS_RAW].cycle_last = tk->tkr_raw.cycle_last;
#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
vc[CS_RAW].max_cycles = tk->tkr_raw.clock->max_cycles;
#endif
vc[CS_RAW].mask = tk->tkr_raw.mask;
vc[CS_RAW].mult = tk->tkr_raw.mult;
vc[CS_RAW].shift = tk->tkr_raw.shift;
fill_clock_configuration(&vc[CS_HRES_COARSE], &tk->tkr_mono);
fill_clock_configuration(&vc[CS_RAW], &tk->tkr_raw);
/* CLOCK_MONOTONIC */
vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC];
@@ -119,7 +118,8 @@ void update_vsyscall(struct timekeeper *tk)
if (clock_mode != VDSO_CLOCKMODE_NONE)
update_vdso_time_data(vdata, tk);
__arch_update_vsyscall(vdata);
__arch_update_vdso_clock(&vc[CS_HRES_COARSE]);
__arch_update_vdso_clock(&vc[CS_RAW]);
vdso_write_end(vdata);
@@ -136,6 +136,46 @@ void update_vsyscall_tz(void)
__arch_sync_vdso_time_data(vdata);
}
#ifdef CONFIG_POSIX_AUX_CLOCKS
void vdso_time_update_aux(struct timekeeper *tk)
{
struct vdso_time_data *vdata = vdso_k_time_data;
struct vdso_timestamp *vdso_ts;
struct vdso_clock *vc;
s32 clock_mode;
u64 nsec;
vc = &vdata->aux_clock_data[tk->id - TIMEKEEPER_AUX_FIRST];
vdso_ts = &vc->basetime[VDSO_BASE_AUX];
clock_mode = tk->tkr_mono.clock->vdso_clock_mode;
if (!tk->clock_valid)
clock_mode = VDSO_CLOCKMODE_NONE;
/* copy vsyscall data */
vdso_write_begin_clock(vc);
vc->clock_mode = clock_mode;
if (clock_mode != VDSO_CLOCKMODE_NONE) {
fill_clock_configuration(vc, &tk->tkr_mono);
vdso_ts->sec = tk->xtime_sec + tk->monotonic_to_aux.tv_sec;
nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
nsec += tk->monotonic_to_aux.tv_nsec;
vdso_ts->sec += __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
nsec = nsec << tk->tkr_mono.shift;
vdso_ts->nsec = nsec;
}
__arch_update_vdso_clock(vc);
vdso_write_end_clock(vc);
__arch_sync_vdso_time_data(vdata);
}
#endif
/**
* vdso_update_begin - Start of a VDSO update section
*
kernel/utsname.c
+1 -1
View File
@@ -86,7 +86,7 @@ fail:
* utsname of this process won't be seen by parent, and vice
* versa.
*/
struct uts_namespace *copy_utsname(unsigned long flags,
struct uts_namespace *copy_utsname(u64 flags,
struct user_namespace *user_ns, struct uts_namespace *old_ns)
{
struct uts_namespace *new_ns;
lib/find_bit.c
+24
View File
@@ -18,6 +18,7 @@
#include <linux/math.h>
#include <linux/minmax.h>
#include <linux/swab.h>
#include <linux/random.h>
/*
* Common helper for find_bit() function family
@@ -291,3 +292,26 @@ EXPORT_SYMBOL(_find_next_bit_le);
#endif
#endif /* __BIG_ENDIAN */
/**
* find_random_bit - find a set bit at random position
* @addr: The address to base the search on
* @size: The bitmap size in bits
*
* Returns: a position of a random set bit; >= @size otherwise
*/
unsigned long find_random_bit(const unsigned long *addr, unsigned long size)
{
int w = bitmap_weight(addr, size);
switch (w) {
case 0:
return size;
case 1:
/* Performance trick for single-bit bitmaps */
return find_first_bit(addr, size);
default:
return find_nth_bit(addr, size, get_random_u32_below(w));
}
}
EXPORT_SYMBOL(find_random_bit);
lib/test_objpool.c
+1 -1
View File
@@ -164,7 +164,7 @@ static enum hrtimer_restart ot_hrtimer_handler(struct hrtimer *hrt)
/* do bulk-testings for objects pop/push */
item->worker(item, 1);
hrtimer_forward(hrt, hrt->base->get_time(), item->hrtcycle);
hrtimer_forward_now(hrt, item->hrtcycle);
return HRTIMER_RESTART;
}
net/core/net_namespace.c
+1 -1
View File
@@ -482,7 +482,7 @@ void net_drop_ns(void *p)
net_free(net);
}
struct net *copy_net_ns(unsigned long flags,
struct net *copy_net_ns(u64 flags,
struct user_namespace *user_ns, struct net *old_net)
{
struct ucounts *ucounts;
redhat/configs/rhel/generic/CONFIG_POSIX_AUX_CLOCKS
+1
View File
@@ -0,0 +1 @@
# CONFIG_POSIX_AUX_CLOCKS is not set
scripts/gdb/linux/timerlist.py
-2
View File
@@ -56,8 +56,6 @@ def print_base(base):
text += " .index: {}\n".format(base['index'])
text += " .resolution: {} nsecs\n".format(constants.LX_hrtimer_resolution)
text += " .get_time: {}\n".format(base['get_time'])
if constants.LX_CONFIG_HIGH_RES_TIMERS:
text += " .offset: {} nsecs\n".format(base['offset'])
text += "active timers:\n"
security/apparmor/lsm.c
+1 -1
View File
@@ -111,7 +111,7 @@ static void apparmor_task_free(struct task_struct *task)
}
static int apparmor_task_alloc(struct task_struct *task,
unsigned long clone_flags)
u64 clone_flags)
{
struct aa_task_ctx *new = task_ctx(task);
security/security.c
+1 -1
View File
@@ -3159,7 +3159,7 @@ int security_file_truncate(struct file *file)
*
* Return: Returns a zero on success, negative values on failure.
*/
int security_task_alloc(struct task_struct *task, unsigned long clone_flags)
int security_task_alloc(struct task_struct *task, u64 clone_flags)
{
int rc = lsm_task_alloc(task);
security/selinux/hooks.c
+1 -1
View File
@@ -4001,7 +4001,7 @@ static int selinux_file_open(struct file *file)
/* task security operations */
static int selinux_task_alloc(struct task_struct *task,
unsigned long clone_flags)
u64 clone_flags)
{
u32 sid = current_sid();
security/tomoyo/tomoyo.c
+1 -1
View File
@@ -514,7 +514,7 @@ struct lsm_blob_sizes tomoyo_blob_sizes __ro_after_init = {
* Returns 0.
*/
static int tomoyo_task_alloc(struct task_struct *task,
unsigned long clone_flags)
u64 clone_flags)
{
struct tomoyo_task *old = tomoyo_task(current);
struct tomoyo_task *new = tomoyo_task(task);