Skip to content

Commit 7b953a5

Browse files
ioworker0ioworker0
committed
hung_task: show the blocker task if the task is hung on semaphore
Inspired by mutex blocker tracking[1], this patch makes a trade-off to balance the overhead and utility of the hung task detector. Unlike mutexes, semaphores lack explicit ownership tracking, making it challenging to identify the root cause of hangs. To address this, we introduce a last_holder field to the semaphore structure, which is updated when a task successfully calls down() and cleared during up(). The assumption is that if a task is blocked on a semaphore, the holders must not have released it. While this does not guarantee that the last holder is one of the current blockers, it likely provides a practical hint for diagnosing semaphore-related stalls. With this change, the hung task detector can now show blocker task's info like below: [Thu Mar 13 15:18:38 2025] INFO: task cat:1803 blocked for more than 122 seconds. [Thu Mar 13 15:18:38 2025] Tainted: G OE 6.14.0-rc3+ torvalds#14 [Thu Mar 13 15:18:38 2025] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Thu Mar 13 15:18:38 2025] task:cat state:D stack:0 pid:1803 tgid:1803 ppid:1057 task_flags:0x400000 flags:0x00000004 [Thu Mar 13 15:18:38 2025] Call trace: [Thu Mar 13 15:18:38 2025] __switch_to+0x1ec/0x380 (T) [Thu Mar 13 15:18:38 2025] __schedule+0xc30/0x44f8 [Thu Mar 13 15:18:38 2025] schedule+0xb8/0x3b0 [Thu Mar 13 15:18:38 2025] schedule_timeout+0x1d0/0x208 [Thu Mar 13 15:18:38 2025] __down_common+0x2d4/0x6f8 [Thu Mar 13 15:18:38 2025] __down+0x24/0x50 [Thu Mar 13 15:18:38 2025] down+0xd0/0x140 [Thu Mar 13 15:18:38 2025] read_dummy+0x3c/0xa0 [hung_task_sem] [Thu Mar 13 15:18:38 2025] full_proxy_read+0xfc/0x1d0 [Thu Mar 13 15:18:38 2025] vfs_read+0x1a0/0x858 [Thu Mar 13 15:18:38 2025] ksys_read+0x100/0x220 [Thu Mar 13 15:18:38 2025] __arm64_sys_read+0x78/0xc8 [Thu Mar 13 15:18:38 2025] invoke_syscall+0xd8/0x278 [Thu Mar 13 15:18:38 2025] el0_svc_common.constprop.0+0xb8/0x298 [Thu Mar 13 15:18:38 2025] do_el0_svc+0x4c/0x88 [Thu Mar 13 15:18:38 2025] el0_svc+0x44/0x108 [Thu Mar 13 15:18:38 2025] el0t_64_sync_handler+0x134/0x160 [Thu Mar 13 15:18:38 2025] el0t_64_sync+0x1b8/0x1c0 [Thu Mar 13 15:18:38 2025] INFO: task cat:1803 blocked on a semaphore likely last held by task cat:1802 [Thu Mar 13 15:18:38 2025] task:cat state:S stack:0 pid:1802 tgid:1802 ppid:1057 task_flags:0x400000 flags:0x00000004 [Thu Mar 13 15:18:38 2025] Call trace: [Thu Mar 13 15:18:38 2025] __switch_to+0x1ec/0x380 (T) [Thu Mar 13 15:18:38 2025] __schedule+0xc30/0x44f8 [Thu Mar 13 15:18:38 2025] schedule+0xb8/0x3b0 [Thu Mar 13 15:18:38 2025] schedule_timeout+0xf4/0x208 [Thu Mar 13 15:18:38 2025] msleep_interruptible+0x70/0x130 [Thu Mar 13 15:18:38 2025] read_dummy+0x48/0xa0 [hung_task_sem] [Thu Mar 13 15:18:38 2025] full_proxy_read+0xfc/0x1d0 [Thu Mar 13 15:18:38 2025] vfs_read+0x1a0/0x858 [Thu Mar 13 15:18:38 2025] ksys_read+0x100/0x220 [Thu Mar 13 15:18:38 2025] __arm64_sys_read+0x78/0xc8 [Thu Mar 13 15:18:38 2025] invoke_syscall+0xd8/0x278 [Thu Mar 13 15:18:38 2025] el0_svc_common.constprop.0+0xb8/0x298 [Thu Mar 13 15:18:38 2025] do_el0_svc+0x4c/0x88 [Thu Mar 13 15:18:38 2025] el0_svc+0x44/0x108 [Thu Mar 13 15:18:38 2025] el0t_64_sync_handler+0x134/0x160 [Thu Mar 13 15:18:38 2025] el0t_64_sync+0x1b8/0x1c0 [1] https://lore.kernel.org/all/174046694331.2194069.15472952050240807469.stgit@mhiramat.tok.corp.google.com Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org> Signed-off-by: Mingzhe Yang <mingzhe.yang@ly.com> Signed-off-by: Lance Yang <ioworker0@gmail.com>
1 parent 36f6dc3 commit 7b953a5

File tree

3 files changed

+98
-17
lines changed

3 files changed

+98
-17
lines changed

‎include/linux/semaphore.h

+14-1
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,25 @@ struct semaphore {
1616
raw_spinlock_t lock;
1717
unsigned int count;
1818
struct list_head wait_list;
19+
20+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
21+
unsigned long last_holder;
22+
#endif
1923
};
2024

25+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
26+
#define __LAST_HOLDER_SEMAPHORE_INITIALIZER \
27+
, .last_holder = 0UL
28+
#else
29+
#define __LAST_HOLDER_SEMAPHORE_INITIALIZER
30+
#endif
31+
2132
#define __SEMAPHORE_INITIALIZER(name, n) \
2233
{ \
2334
.lock = __RAW_SPIN_LOCK_UNLOCKED((name).lock), \
2435
.count = n, \
25-
.wait_list = LIST_HEAD_INIT((name).wait_list), \
36+
.wait_list = LIST_HEAD_INIT((name).wait_list) \
37+
__LAST_HOLDER_SEMAPHORE_INITIALIZER \
2638
}
2739

2840
/*
@@ -47,5 +59,6 @@ extern int __must_check down_killable(struct semaphore *sem);
4759
extern int __must_check down_trylock(struct semaphore *sem);
4860
extern int __must_check down_timeout(struct semaphore *sem, long jiffies);
4961
extern void up(struct semaphore *sem);
62+
extern unsigned long sem_last_holder(struct semaphore *sem);
5063

5164
#endif /* __LINUX_SEMAPHORE_H */

‎kernel/hung_task.c

+35-10
Original file line numberDiff line numberDiff line change
@@ -102,31 +102,56 @@ static struct notifier_block panic_block = {
102102
static void debug_show_blocker(struct task_struct *task)
103103
{
104104
struct task_struct *g, *t;
105-
unsigned long owner, blocker;
105+
unsigned long owner, blocker, blocker_lock_type;
106106

107107
RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "No rcu lock held");
108108

109109
blocker = READ_ONCE(task->blocker);
110-
if (!blocker || !hung_task_blocker_is_type(blocker, BLOCKER_TYPE_MUTEX))
110+
if (!blocker)
111111
return;
112112

113-
owner = mutex_get_owner(
114-
(struct mutex *)hung_task_blocker_to_lock(blocker));
113+
if (hung_task_blocker_is_type(blocker, BLOCKER_TYPE_MUTEX)) {
114+
owner = mutex_get_owner(
115+
(struct mutex *)hung_task_blocker_to_lock(blocker));
116+
blocker_lock_type = BLOCKER_TYPE_MUTEX;
117+
} else if (hung_task_blocker_is_type(blocker, BLOCKER_TYPE_SEM)) {
118+
owner = sem_last_holder(
119+
(struct semaphore *)hung_task_blocker_to_lock(blocker));
120+
blocker_lock_type = BLOCKER_TYPE_SEM;
121+
} else
122+
return;
115123

116124
if (unlikely(!owner)) {
117-
pr_err("INFO: task %s:%d is blocked on a mutex, but the owner is not found.\n",
118-
task->comm, task->pid);
125+
switch (blocker_lock_type) {
126+
case BLOCKER_TYPE_MUTEX:
127+
pr_err("INFO: task %s:%d is blocked on a mutex, but the owner is not found.\n",
128+
task->comm, task->pid);
129+
break;
130+
case BLOCKER_TYPE_SEM:
131+
pr_err("INFO: task %s:%d is blocked on a semaphore, but the last holder is not found.\n",
132+
task->comm, task->pid);
133+
break;
134+
}
119135
return;
120136
}
121137

122138
/* Ensure the owner information is correct. */
123139
for_each_process_thread(g, t) {
124-
if ((unsigned long)t == owner) {
140+
if ((unsigned long)t != owner)
141+
continue;
142+
143+
switch (blocker_lock_type) {
144+
case BLOCKER_TYPE_MUTEX:
125145
pr_err("INFO: task %s:%d is blocked on a mutex likely owned by task %s:%d.\n",
126-
task->comm, task->pid, t->comm, t->pid);
127-
sched_show_task(t);
128-
return;
146+
task->comm, task->pid, t->comm, t->pid);
147+
break;
148+
case BLOCKER_TYPE_SEM:
149+
pr_err("INFO: task %s:%d blocked on a semaphore likely last held by task %s:%d\n",
150+
task->comm, task->pid, t->comm, t->pid);
151+
break;
129152
}
153+
sched_show_task(t);
154+
return;
130155
}
131156
}
132157
#else

‎kernel/locking/semaphore.c

+49-6
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,16 @@
3434
#include <linux/ftrace.h>
3535
#include <trace/events/lock.h>
3636

37+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
38+
#include <linux/hung_task.h>
39+
#endif
40+
3741
static noinline void __down(struct semaphore *sem);
3842
static noinline int __down_interruptible(struct semaphore *sem);
3943
static noinline int __down_killable(struct semaphore *sem);
4044
static noinline int __down_timeout(struct semaphore *sem, long timeout);
4145
static noinline void __up(struct semaphore *sem);
46+
static inline void __sem_acquire(struct semaphore *sem);
4247

4348
/**
4449
* down - acquire the semaphore
@@ -58,7 +63,7 @@ void __sched down(struct semaphore *sem)
5863
might_sleep();
5964
raw_spin_lock_irqsave(&sem->lock, flags);
6065
if (likely(sem->count > 0))
61-
sem->count--;
66+
__sem_acquire(sem);
6267
else
6368
__down(sem);
6469
raw_spin_unlock_irqrestore(&sem->lock, flags);
@@ -82,7 +87,7 @@ int __sched down_interruptible(struct semaphore *sem)
8287
might_sleep();
8388
raw_spin_lock_irqsave(&sem->lock, flags);
8489
if (likely(sem->count > 0))
85-
sem->count--;
90+
__sem_acquire(sem);
8691
else
8792
result = __down_interruptible(sem);
8893
raw_spin_unlock_irqrestore(&sem->lock, flags);
@@ -109,7 +114,7 @@ int __sched down_killable(struct semaphore *sem)
109114
might_sleep();
110115
raw_spin_lock_irqsave(&sem->lock, flags);
111116
if (likely(sem->count > 0))
112-
sem->count--;
117+
__sem_acquire(sem);
113118
else
114119
result = __down_killable(sem);
115120
raw_spin_unlock_irqrestore(&sem->lock, flags);
@@ -139,7 +144,7 @@ int __sched down_trylock(struct semaphore *sem)
139144
raw_spin_lock_irqsave(&sem->lock, flags);
140145
count = sem->count - 1;
141146
if (likely(count >= 0))
142-
sem->count = count;
147+
__sem_acquire(sem);
143148
raw_spin_unlock_irqrestore(&sem->lock, flags);
144149

145150
return (count < 0);
@@ -164,7 +169,7 @@ int __sched down_timeout(struct semaphore *sem, long timeout)
164169
might_sleep();
165170
raw_spin_lock_irqsave(&sem->lock, flags);
166171
if (likely(sem->count > 0))
167-
sem->count--;
172+
__sem_acquire(sem);
168173
else
169174
result = __down_timeout(sem, timeout);
170175
raw_spin_unlock_irqrestore(&sem->lock, flags);
@@ -185,6 +190,12 @@ void __sched up(struct semaphore *sem)
185190
unsigned long flags;
186191

187192
raw_spin_lock_irqsave(&sem->lock, flags);
193+
194+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
195+
if (READ_ONCE(sem->last_holder) == (unsigned long)current)
196+
WRITE_ONCE(sem->last_holder, 0UL);
197+
#endif
198+
188199
if (likely(list_empty(&sem->wait_list)))
189200
sem->count++;
190201
else
@@ -224,8 +235,12 @@ static inline int __sched ___down_common(struct semaphore *sem, long state,
224235
raw_spin_unlock_irq(&sem->lock);
225236
timeout = schedule_timeout(timeout);
226237
raw_spin_lock_irq(&sem->lock);
227-
if (waiter.up)
238+
if (waiter.up) {
239+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
240+
WRITE_ONCE(sem->last_holder, (unsigned long)current);
241+
#endif
228242
return 0;
243+
}
229244
}
230245

231246
timed_out:
@@ -242,10 +257,18 @@ static inline int __sched __down_common(struct semaphore *sem, long state,
242257
{
243258
int ret;
244259

260+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
261+
hung_task_set_blocker(sem, BLOCKER_TYPE_SEM);
262+
#endif
263+
245264
trace_contention_begin(sem, 0);
246265
ret = ___down_common(sem, state, timeout);
247266
trace_contention_end(sem, ret);
248267

268+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
269+
hung_task_clear_blocker();
270+
#endif
271+
249272
return ret;
250273
}
251274

@@ -277,3 +300,23 @@ static noinline void __sched __up(struct semaphore *sem)
277300
waiter->up = true;
278301
wake_up_process(waiter->task);
279302
}
303+
304+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
305+
unsigned long sem_last_holder(struct semaphore *sem)
306+
{
307+
return READ_ONCE(sem->last_holder);
308+
}
309+
#else
310+
unsigned long sem_last_holder(struct semaphore *sem)
311+
{
312+
return 0UL;
313+
}
314+
#endif
315+
316+
static inline void __sem_acquire(struct semaphore *sem)
317+
{
318+
sem->count--;
319+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
320+
WRITE_ONCE(sem->last_holder, (unsigned long)current);
321+
#endif
322+
}

0 commit comments

Comments
 (0)