linux/include/linux/alloc_tag.h at multikernel-arm · multikernel/linux · GitHub

History

268 lines (213 loc) · 6.87 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

/* SPDX-License-Identifier: GPL-2.0 */

* allocation tagging

#ifndef _LINUX_ALLOC_TAG_H

#define _LINUX_ALLOC_TAG_H

#include <linux/bug.h>

#include <linux/codetag.h>

#include <linux/container_of.h>

#include <linux/preempt.h>

#include <asm/percpu.h>

#include <linux/cpumask.h>

#include <linux/smp.h>

#include <linux/static_key.h>

#include <linux/irqflags.h>

struct alloc_tag_counters {

u64 bytes;

u64 calls;

};

* An instance of this structure is created in a special ELF section at every

* allocation callsite. At runtime, the special section is treated as

* an array of these. Embedded codetag utilizes codetag framework.

struct alloc_tag {

struct codetag ct;

struct alloc_tag_counters __percpu *counters;

} __aligned(8);

struct alloc_tag_kernel_section {

struct alloc_tag *first_tag;

unsigned long count;

};

struct alloc_tag_module_section {

union {

unsigned long start_addr;

struct alloc_tag *first_tag;

};

unsigned long end_addr;

/* used size */

unsigned long size;

};

#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG

#define CODETAG_EMPTY ((void *)1)

static inline bool is_codetag_empty(union codetag_ref *ref)

{

return ref->ct == CODETAG_EMPTY;

}

static inline void set_codetag_empty(union codetag_ref *ref)

{

if (ref)

ref->ct = CODETAG_EMPTY;

}

#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */

static inline bool is_codetag_empty(union codetag_ref *ref) { return false; }

static inline void set_codetag_empty(union codetag_ref *ref)

{

if (ref)

ref->ct = NULL;

}

#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */

#ifdef CONFIG_MEM_ALLOC_PROFILING

#define ALLOC_TAG_SECTION_NAME "alloc_tags"

struct codetag_bytes {

struct codetag *ct;

s64 bytes;

};

size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep);

static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct)

{

return container_of(ct, struct alloc_tag, ct);

}

#if defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE)

* When percpu variables are required to be defined as weak, static percpu

* variables can't be used inside a function (see comments for DECLARE_PER_CPU_SECTION).

* Instead we will account all module allocations to a single counter.

DECLARE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);

#define DEFINE_ALLOC_TAG(_alloc_tag) \

static struct alloc_tag _alloc_tag __used __aligned(8) \

__section(ALLOC_TAG_SECTION_NAME) = { \

.ct = CODE_TAG_INIT, \

.counters = &_shared_alloc_tag };

#else /* CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU && MODULE */

#ifdef MODULE

#define DEFINE_ALLOC_TAG(_alloc_tag) \

static struct alloc_tag _alloc_tag __used __aligned(8) \

__section(ALLOC_TAG_SECTION_NAME) = { \

.ct = CODE_TAG_INIT, \

.counters = NULL };

#else /* MODULE */

#define DEFINE_ALLOC_TAG(_alloc_tag) \

static DEFINE_PER_CPU(struct alloc_tag_counters, _alloc_tag_cntr); \

static struct alloc_tag _alloc_tag __used __aligned(8) \

__section(ALLOC_TAG_SECTION_NAME) = { \

.ct = CODE_TAG_INIT, \

.counters = &_alloc_tag_cntr };

#endif /* MODULE */

#endif /* CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU && MODULE */

DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,

mem_alloc_profiling_key);

static inline bool mem_alloc_profiling_enabled(void)

{

return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,

&mem_alloc_profiling_key);

}

static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag)

{

struct alloc_tag_counters v = { 0, 0 };

struct alloc_tag_counters *counter;

int cpu;

for_each_possible_cpu(cpu) {

counter = per_cpu_ptr(tag->counters, cpu);

v.bytes += counter->bytes;

v.calls += counter->calls;

}

return v;

}

#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG

static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag)

{

WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),

"alloc_tag was not cleared (got tag for %s:%u)\n",

ref->ct->filename, ref->ct->lineno);

WARN_ONCE(!tag, "current->alloc_tag not set\n");

}

static inline void alloc_tag_sub_check(union codetag_ref *ref)

{

WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n");

}

#else

static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {}

static inline void alloc_tag_sub_check(union codetag_ref *ref) {}

#endif

/* Caller should verify both ref and tag to be valid */

static inline bool __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)

{

alloc_tag_add_check(ref, tag);

if (!ref || !tag)

return false;

ref->ct = &tag->ct;

return true;

}

static inline bool alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)

{

if (unlikely(!__alloc_tag_ref_set(ref, tag)))

return false;

* We need in increment the call counter every time we have a new

* allocation or when we split a large allocation into smaller ones.

* Each new reference for every sub-allocation needs to increment call

* counter because when we free each part the counter will be decremented.

this_cpu_inc(tag->counters->calls);

return true;

}

static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes)

{

if (likely(alloc_tag_ref_set(ref, tag)))

this_cpu_add(tag->counters->bytes, bytes);

}

static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes)

{

struct alloc_tag *tag;

alloc_tag_sub_check(ref);

if (!ref || !ref->ct)

return;

if (is_codetag_empty(ref)) {

ref->ct = NULL;

return;

}

tag = ct_to_alloc_tag(ref->ct);

this_cpu_sub(tag->counters->bytes, bytes);

this_cpu_dec(tag->counters->calls);

ref->ct = NULL;

}

static inline void alloc_tag_set_inaccurate(struct alloc_tag *tag)

{

tag->ct.flags |= CODETAG_FLAG_INACCURATE;

}

static inline bool alloc_tag_is_inaccurate(struct alloc_tag *tag)

{

return !!(tag->ct.flags & CODETAG_FLAG_INACCURATE);

}

#define alloc_tag_record(p) ((p) = current->alloc_tag)

#else /* CONFIG_MEM_ALLOC_PROFILING */

#define DEFINE_ALLOC_TAG(_alloc_tag)

static inline bool mem_alloc_profiling_enabled(void) { return false; }

static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag,

size_t bytes) {}

static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {}

static inline void alloc_tag_set_inaccurate(struct alloc_tag *tag) {}

static inline bool alloc_tag_is_inaccurate(struct alloc_tag *tag) { return false; }

#define alloc_tag_record(p) do {} while (0)

#endif /* CONFIG_MEM_ALLOC_PROFILING */

#define alloc_hooks_tag(_tag, _do_alloc) \

({ \

typeof(_do_alloc) _res; \

if (mem_alloc_profiling_enabled()) { \

struct alloc_tag * __maybe_unused _old; \

_old = alloc_tag_save(_tag); \

_res = _do_alloc; \

alloc_tag_restore(_tag, _old); \

} else \

_res = _do_alloc; \

_res; \

})

#define alloc_hooks(_do_alloc) \

({ \

DEFINE_ALLOC_TAG(_alloc_tag); \

alloc_hooks_tag(&_alloc_tag, _do_alloc); \

})

#endif /* _LINUX_ALLOC_TAG_H */

linux/include/linux/alloc_tag.h at multikernel-arm · multikernel/linux · GitHub

Expand file tree

Breadcrumbs

alloc_tag.h

Latest commit

History

Breadcrumbs

alloc_tag.h

File metadata and controls