Skip to content

Commit 1a52728

Browse files
kvaneeshozbenh
kvaneesh
authored and
ozbenh
committed
powerpc: Optimize hugepage invalidate
Hugepage invalidate involves invalidating multiple hpte entries. Optimize the operation using H_BULK_REMOVE on lpar platforms. On native, reduce the number of tlb flush. Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
1 parent 437d496 commit 1a52728

File tree

4 files changed

+201
-9
lines changed

4 files changed

+201
-9
lines changed

‎arch/powerpc/include/asm/machdep.h

+3
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ struct machdep_calls {
5757
void (*hpte_removebolted)(unsigned long ea,
5858
int psize, int ssize);
5959
void (*flush_hash_range)(unsigned long number, int local);
60+
void (*hugepage_invalidate)(struct mm_struct *mm,
61+
unsigned char *hpte_slot_array,
62+
unsigned long addr, int psize);
6063

6164
/* special for kexec, to be called in real mode, linear mapping is
6265
* destroyed as well */

‎arch/powerpc/mm/hash_native_64.c

+73
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,78 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
407407
local_irq_restore(flags);
408408
}
409409

410+
static void native_hugepage_invalidate(struct mm_struct *mm,
411+
unsigned char *hpte_slot_array,
412+
unsigned long addr, int psize)
413+
{
414+
int ssize = 0, i;
415+
int lock_tlbie;
416+
struct hash_pte *hptep;
417+
int actual_psize = MMU_PAGE_16M;
418+
unsigned int max_hpte_count, valid;
419+
unsigned long flags, s_addr = addr;
420+
unsigned long hpte_v, want_v, shift;
421+
unsigned long hidx, vpn = 0, vsid, hash, slot;
422+
423+
shift = mmu_psize_defs[psize].shift;
424+
max_hpte_count = 1U << (PMD_SHIFT - shift);
425+
426+
local_irq_save(flags);
427+
for (i = 0; i < max_hpte_count; i++) {
428+
valid = hpte_valid(hpte_slot_array, i);
429+
if (!valid)
430+
continue;
431+
hidx = hpte_hash_index(hpte_slot_array, i);
432+
433+
/* get the vpn */
434+
addr = s_addr + (i * (1ul << shift));
435+
if (!is_kernel_addr(addr)) {
436+
ssize = user_segment_size(addr);
437+
vsid = get_vsid(mm->context.id, addr, ssize);
438+
WARN_ON(vsid == 0);
439+
} else {
440+
vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
441+
ssize = mmu_kernel_ssize;
442+
}
443+
444+
vpn = hpt_vpn(addr, vsid, ssize);
445+
hash = hpt_hash(vpn, shift, ssize);
446+
if (hidx & _PTEIDX_SECONDARY)
447+
hash = ~hash;
448+
449+
slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
450+
slot += hidx & _PTEIDX_GROUP_IX;
451+
452+
hptep = htab_address + slot;
453+
want_v = hpte_encode_avpn(vpn, psize, ssize);
454+
native_lock_hpte(hptep);
455+
hpte_v = hptep->v;
456+
457+
/* Even if we miss, we need to invalidate the TLB */
458+
if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
459+
native_unlock_hpte(hptep);
460+
else
461+
/* Invalidate the hpte. NOTE: this also unlocks it */
462+
hptep->v = 0;
463+
}
464+
/*
465+
* Since this is a hugepage, we just need a single tlbie.
466+
* use the last vpn.
467+
*/
468+
lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
469+
if (lock_tlbie)
470+
raw_spin_lock(&native_tlbie_lock);
471+
472+
asm volatile("ptesync":::"memory");
473+
__tlbie(vpn, psize, actual_psize, ssize);
474+
asm volatile("eieio; tlbsync; ptesync":::"memory");
475+
476+
if (lock_tlbie)
477+
raw_spin_unlock(&native_tlbie_lock);
478+
479+
local_irq_restore(flags);
480+
}
481+
410482
static inline int __hpte_actual_psize(unsigned int lp, int psize)
411483
{
412484
int i, shift;
@@ -640,4 +712,5 @@ void __init hpte_init_native(void)
640712
ppc_md.hpte_remove = native_hpte_remove;
641713
ppc_md.hpte_clear_all = native_hpte_clear;
642714
ppc_md.flush_hash_range = native_flush_hash_range;
715+
ppc_md.hugepage_invalidate = native_hugepage_invalidate;
643716
}

‎arch/powerpc/mm/pgtable_64.c

+10-2
Original file line numberDiff line numberDiff line change
@@ -708,6 +708,7 @@ void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
708708
{
709709
int ssize, i;
710710
unsigned long s_addr;
711+
int max_hpte_count;
711712
unsigned int psize, valid;
712713
unsigned char *hpte_slot_array;
713714
unsigned long hidx, vpn, vsid, hash, shift, slot;
@@ -727,9 +728,16 @@ void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
727728

728729
/* get the base page size */
729730
psize = get_slice_psize(mm, s_addr);
730-
shift = mmu_psize_defs[psize].shift;
731731

732-
for (i = 0; i < (HPAGE_PMD_SIZE >> shift); i++) {
732+
if (ppc_md.hugepage_invalidate)
733+
return ppc_md.hugepage_invalidate(mm, hpte_slot_array,
734+
s_addr, psize);
735+
/*
736+
* No bluk hpte removal support, invalidate each entry
737+
*/
738+
shift = mmu_psize_defs[psize].shift;
739+
max_hpte_count = HPAGE_PMD_SIZE >> shift;
740+
for (i = 0; i < max_hpte_count; i++) {
733741
/*
734742
* 8 bits per each hpte entries
735743
* 000| [ secondary group (one bit) | hidx (3 bits) | valid bit]

‎arch/powerpc/platforms/pseries/lpar.c

+115-7
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,13 @@
4545
#include "plpar_wrappers.h"
4646
#include "pseries.h"
4747

48+
/* Flag bits for H_BULK_REMOVE */
49+
#define HBR_REQUEST 0x4000000000000000UL
50+
#define HBR_RESPONSE 0x8000000000000000UL
51+
#define HBR_END 0xc000000000000000UL
52+
#define HBR_AVPN 0x0200000000000000UL
53+
#define HBR_ANDCOND 0x0100000000000000UL
54+
4855

4956
/* in hvCall.S */
5057
EXPORT_SYMBOL(plpar_hcall);
@@ -347,6 +354,113 @@ static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,
347354
BUG_ON(lpar_rc != H_SUCCESS);
348355
}
349356

357+
/*
358+
* Limit iterations holding pSeries_lpar_tlbie_lock to 3. We also need
359+
* to make sure that we avoid bouncing the hypervisor tlbie lock.
360+
*/
361+
#define PPC64_HUGE_HPTE_BATCH 12
362+
363+
static void __pSeries_lpar_hugepage_invalidate(unsigned long *slot,
364+
unsigned long *vpn, int count,
365+
int psize, int ssize)
366+
{
367+
unsigned long param[8];
368+
int i = 0, pix = 0, rc;
369+
unsigned long flags = 0;
370+
int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
371+
372+
if (lock_tlbie)
373+
spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags);
374+
375+
for (i = 0; i < count; i++) {
376+
377+
if (!firmware_has_feature(FW_FEATURE_BULK_REMOVE)) {
378+
pSeries_lpar_hpte_invalidate(slot[i], vpn[i], psize, 0,
379+
ssize, 0);
380+
} else {
381+
param[pix] = HBR_REQUEST | HBR_AVPN | slot[i];
382+
param[pix+1] = hpte_encode_avpn(vpn[i], psize, ssize);
383+
pix += 2;
384+
if (pix == 8) {
385+
rc = plpar_hcall9(H_BULK_REMOVE, param,
386+
param[0], param[1], param[2],
387+
param[3], param[4], param[5],
388+
param[6], param[7]);
389+
BUG_ON(rc != H_SUCCESS);
390+
pix = 0;
391+
}
392+
}
393+
}
394+
if (pix) {
395+
param[pix] = HBR_END;
396+
rc = plpar_hcall9(H_BULK_REMOVE, param, param[0], param[1],
397+
param[2], param[3], param[4], param[5],
398+
param[6], param[7]);
399+
BUG_ON(rc != H_SUCCESS);
400+
}
401+
402+
if (lock_tlbie)
403+
spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags);
404+
}
405+
406+
static void pSeries_lpar_hugepage_invalidate(struct mm_struct *mm,
407+
unsigned char *hpte_slot_array,
408+
unsigned long addr, int psize)
409+
{
410+
int ssize = 0, i, index = 0;
411+
unsigned long s_addr = addr;
412+
unsigned int max_hpte_count, valid;
413+
unsigned long vpn_array[PPC64_HUGE_HPTE_BATCH];
414+
unsigned long slot_array[PPC64_HUGE_HPTE_BATCH];
415+
unsigned long shift, hidx, vpn = 0, vsid, hash, slot;
416+
417+
shift = mmu_psize_defs[psize].shift;
418+
max_hpte_count = 1U << (PMD_SHIFT - shift);
419+
420+
for (i = 0; i < max_hpte_count; i++) {
421+
valid = hpte_valid(hpte_slot_array, i);
422+
if (!valid)
423+
continue;
424+
hidx = hpte_hash_index(hpte_slot_array, i);
425+
426+
/* get the vpn */
427+
addr = s_addr + (i * (1ul << shift));
428+
if (!is_kernel_addr(addr)) {
429+
ssize = user_segment_size(addr);
430+
vsid = get_vsid(mm->context.id, addr, ssize);
431+
WARN_ON(vsid == 0);
432+
} else {
433+
vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
434+
ssize = mmu_kernel_ssize;
435+
}
436+
437+
vpn = hpt_vpn(addr, vsid, ssize);
438+
hash = hpt_hash(vpn, shift, ssize);
439+
if (hidx & _PTEIDX_SECONDARY)
440+
hash = ~hash;
441+
442+
slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
443+
slot += hidx & _PTEIDX_GROUP_IX;
444+
445+
slot_array[index] = slot;
446+
vpn_array[index] = vpn;
447+
if (index == PPC64_HUGE_HPTE_BATCH - 1) {
448+
/*
449+
* Now do a bluk invalidate
450+
*/
451+
__pSeries_lpar_hugepage_invalidate(slot_array,
452+
vpn_array,
453+
PPC64_HUGE_HPTE_BATCH,
454+
psize, ssize);
455+
index = 0;
456+
} else
457+
index++;
458+
}
459+
if (index)
460+
__pSeries_lpar_hugepage_invalidate(slot_array, vpn_array,
461+
index, psize, ssize);
462+
}
463+
350464
static void pSeries_lpar_hpte_removebolted(unsigned long ea,
351465
int psize, int ssize)
352466
{
@@ -364,13 +478,6 @@ static void pSeries_lpar_hpte_removebolted(unsigned long ea,
364478
pSeries_lpar_hpte_invalidate(slot, vpn, psize, 0, ssize, 0);
365479
}
366480

367-
/* Flag bits for H_BULK_REMOVE */
368-
#define HBR_REQUEST 0x4000000000000000UL
369-
#define HBR_RESPONSE 0x8000000000000000UL
370-
#define HBR_END 0xc000000000000000UL
371-
#define HBR_AVPN 0x0200000000000000UL
372-
#define HBR_ANDCOND 0x0100000000000000UL
373-
374481
/*
375482
* Take a spinlock around flushes to avoid bouncing the hypervisor tlbie
376483
* lock.
@@ -459,6 +566,7 @@ void __init hpte_init_lpar(void)
459566
ppc_md.hpte_removebolted = pSeries_lpar_hpte_removebolted;
460567
ppc_md.flush_hash_range = pSeries_lpar_flush_hash_range;
461568
ppc_md.hpte_clear_all = pSeries_lpar_hptab_clear;
569+
ppc_md.hugepage_invalidate = pSeries_lpar_hugepage_invalidate;
462570
}
463571

464572
#ifdef CONFIG_PPC_SMLPAR

0 commit comments

Comments
 (0)