Coverage Report

Created: 2022-01-15 10:30

/Users/buildslave/jenkins/workspace/coverage/llvm-project/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- PythonDataObjects.cpp ---------------------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "lldb/Host/Config.h"
10
11
#if LLDB_ENABLE_PYTHON
12
13
#include "PythonDataObjects.h"
14
#include "ScriptInterpreterPython.h"
15
16
#include "lldb/Host/File.h"
17
#include "lldb/Host/FileSystem.h"
18
#include "lldb/Interpreter/ScriptInterpreter.h"
19
#include "lldb/Utility/Log.h"
20
#include "lldb/Utility/Stream.h"
21
22
#include "llvm/ADT/StringSwitch.h"
23
#include "llvm/Support/Casting.h"
24
#include "llvm/Support/ConvertUTF.h"
25
#include "llvm/Support/Errno.h"
26
27
#include <cstdio>
28
29
using namespace lldb_private;
30
using namespace lldb;
31
using namespace lldb_private::python;
32
using llvm::cantFail;
33
using llvm::Error;
34
using llvm::Expected;
35
using llvm::Twine;
36
37
854
template <> Expected<bool> python::As<bool>(Expected<PythonObject> &&obj) {
38
854
  if (!obj)
39
1
    return obj.takeError();
40
853
  return obj.get().IsTrue();
41
854
}
42
43
template <>
44
1.04k
Expected<long long> python::As<long long>(Expected<PythonObject> &&obj) {
45
1.04k
  if (!obj)
46
2
    return obj.takeError();
47
1.03k
  return obj->AsLongLong();
48
1.04k
}
49
50
template <>
51
Expected<unsigned long long>
52
55
python::As<unsigned long long>(Expected<PythonObject> &&obj) {
53
55
  if (!obj)
54
0
    return obj.takeError();
55
55
  return obj->AsUnsignedLongLong();
56
55
}
57
58
template <>
59
8
Expected<std::string> python::As<std::string>(Expected<PythonObject> &&obj) {
60
8
  if (!obj)
61
0
    return obj.takeError();
62
8
  PyObject *str_obj = PyObject_Str(obj.get().get());
63
8
  if (!obj)
64
0
    return llvm::make_error<PythonException>();
65
8
  auto str = Take<PythonString>(str_obj);
66
8
  auto utf8 = str.AsUTF8();
67
8
  if (!utf8)
68
0
    return utf8.takeError();
69
8
  return std::string(utf8.get());
70
8
}
71
72
1.03k
Expected<long long> PythonObject::AsLongLong() const {
73
1.03k
  if (!m_py_obj)
74
0
    return nullDeref();
75
#if PY_MAJOR_VERSION < 3
76
  if (!PyLong_Check(m_py_obj)) {
77
    PythonInteger i(PyRefType::Borrowed, m_py_obj);
78
    return i.AsLongLong();
79
  }
80
#endif
81
1.03k
  assert(!PyErr_Occurred());
82
0
  long long r = PyLong_AsLongLong(m_py_obj);
83
1.03k
  if (PyErr_Occurred())
84
1
    return exception();
85
1.03k
  return r;
86
1.03k
}
87
88
55
Expected<long long> PythonObject::AsUnsignedLongLong() const {
89
55
  if (!m_py_obj)
90
0
    return nullDeref();
91
#if PY_MAJOR_VERSION < 3
92
  if (!PyLong_Check(m_py_obj)) {
93
    PythonInteger i(PyRefType::Borrowed, m_py_obj);
94
    return i.AsUnsignedLongLong();
95
  }
96
#endif
97
55
  assert(!PyErr_Occurred());
98
0
  long long r = PyLong_AsUnsignedLongLong(m_py_obj);
99
55
  if (PyErr_Occurred())
100
0
    return exception();
101
55
  return r;
102
55
}
103
104
// wraps on overflow, instead of raising an error.
105
781
Expected<unsigned long long> PythonObject::AsModuloUnsignedLongLong() const {
106
781
  if (!m_py_obj)
107
0
    return nullDeref();
108
#if PY_MAJOR_VERSION < 3
109
  if (!PyLong_Check(m_py_obj)) {
110
    PythonInteger i(PyRefType::Borrowed, m_py_obj);
111
    return i.AsModuloUnsignedLongLong();
112
  }
113
#endif
114
781
  assert(!PyErr_Occurred());
115
0
  unsigned long long r = PyLong_AsUnsignedLongLongMask(m_py_obj);
116
781
  if (PyErr_Occurred())
117
0
    return exception();
118
781
  return r;
119
781
}
120
121
0
void StructuredPythonObject::Serialize(llvm::json::OStream &s) const {
122
0
  s.value(llvm::formatv("Python Obj: {0:X}", GetValue()).str());
123
0
}
124
125
// PythonObject
126
127
0
void PythonObject::Dump(Stream &strm) const {
128
0
  if (m_py_obj) {
129
0
    FILE *file = llvm::sys::RetryAfterSignal(nullptr, ::tmpfile);
130
0
    if (file) {
131
0
      ::PyObject_Print(m_py_obj, file, 0);
132
0
      const long length = ftell(file);
133
0
      if (length) {
134
0
        ::rewind(file);
135
0
        std::vector<char> file_contents(length, '\0');
136
0
        const size_t length_read =
137
0
            ::fread(file_contents.data(), 1, file_contents.size(), file);
138
0
        if (length_read > 0)
139
0
          strm.Write(file_contents.data(), length_read);
140
0
      }
141
0
      ::fclose(file);
142
0
    }
143
0
  } else
144
0
    strm.PutCString("NULL");
145
0
}
146
147
1.73k
PyObjectType PythonObject::GetObjectType() const {
148
1.73k
  if (!IsAllocated())
149
0
    return PyObjectType::None;
150
151
1.73k
  if (PythonModule::Check(m_py_obj))
152
0
    return PyObjectType::Module;
153
1.73k
  if (PythonList::Check(m_py_obj))
154
15
    return PyObjectType::List;
155
1.72k
  if (PythonTuple::Check(m_py_obj))
156
0
    return PyObjectType::Tuple;
157
1.72k
  if (PythonDictionary::Check(m_py_obj))
158
179
    return PyObjectType::Dictionary;
159
1.54k
  if (PythonString::Check(m_py_obj))
160
737
    return PyObjectType::String;
161
806
#if PY_MAJOR_VERSION >= 3
162
806
  if (PythonBytes::Check(m_py_obj))
163
7
    return PyObjectType::Bytes;
164
799
#endif
165
799
  if (PythonByteArray::Check(m_py_obj))
166
1
    return PyObjectType::ByteArray;
167
798
  if (PythonBoolean::Check(m_py_obj))
168
17
    return PyObjectType::Boolean;
169
781
  if (PythonInteger::Check(m_py_obj))
170
781
    return PyObjectType::Integer;
171
0
  if (PythonFile::Check(m_py_obj))
172
0
    return PyObjectType::File;
173
0
  if (PythonCallable::Check(m_py_obj))
174
0
    return PyObjectType::Callable;
175
0
  return PyObjectType::Unknown;
176
0
}
177
178
0
PythonString PythonObject::Repr() const {
179
0
  if (!m_py_obj)
180
0
    return PythonString();
181
0
  PyObject *repr = PyObject_Repr(m_py_obj);
182
0
  if (!repr)
183
0
    return PythonString();
184
0
  return PythonString(PyRefType::Owned, repr);
185
0
}
186
187
1.56k
PythonString PythonObject::Str() const {
188
1.56k
  if (!m_py_obj)
189
0
    return PythonString();
190
1.56k
  PyObject *str = PyObject_Str(m_py_obj);
191
1.56k
  if (!str)
192
0
    return PythonString();
193
1.56k
  return PythonString(PyRefType::Owned, str);
194
1.56k
}
195
196
PythonObject
197
PythonObject::ResolveNameWithDictionary(llvm::StringRef name,
198
492
                                        const PythonDictionary &dict) {
199
492
  size_t dot_pos = name.find('.');
200
492
  llvm::StringRef piece = name.substr(0, dot_pos);
201
492
  PythonObject result = dict.GetItemForKey(PythonString(piece));
202
492
  if (dot_pos == llvm::StringRef::npos) {
203
    // There was no dot, we're done.
204
115
    return result;
205
115
  }
206
207
  // There was a dot.  The remaining portion of the name should be looked up in
208
  // the context of the object that was found in the dictionary.
209
377
  return result.ResolveName(name.substr(dot_pos + 1));
210
492
}
211
212
4.57k
PythonObject PythonObject::ResolveName(llvm::StringRef name) const {
213
  // Resolve the name in the context of the specified object.  If, for example,
214
  // `this` refers to a PyModule, then this will look for `name` in this
215
  // module.  If `this` refers to a PyType, then it will resolve `name` as an
216
  // attribute of that type.  If `this` refers to an instance of an object,
217
  // then it will resolve `name` as the value of the specified field.
218
  //
219
  // This function handles dotted names so that, for example, if `m_py_obj`
220
  // refers to the `sys` module, and `name` == "path.append", then it will find
221
  // the function `sys.path.append`.
222
223
4.57k
  size_t dot_pos = name.find('.');
224
4.57k
  if (dot_pos == llvm::StringRef::npos) {
225
    // No dots in the name, we should be able to find the value immediately as
226
    // an attribute of `m_py_obj`.
227
4.42k
    return GetAttributeValue(name);
228
4.42k
  }
229
230
  // Look up the first piece of the name, and resolve the rest as a child of
231
  // that.
232
155
  PythonObject parent = ResolveName(name.substr(0, dot_pos));
233
155
  if (!parent.IsAllocated())
234
0
    return PythonObject();
235
236
  // Tail recursion.. should be optimized by the compiler
237
155
  return parent.ResolveName(name.substr(dot_pos + 1));
238
155
}
239
240
2
bool PythonObject::HasAttribute(llvm::StringRef attr) const {
241
2
  if (!IsValid())
242
0
    return false;
243
2
  PythonString py_attr(attr);
244
2
  return !!PyObject_HasAttr(m_py_obj, py_attr.get());
245
2
}
246
247
4.42k
PythonObject PythonObject::GetAttributeValue(llvm::StringRef attr) const {
248
4.42k
  if (!IsValid())
249
1
    return PythonObject();
250
251
4.42k
  PythonString py_attr(attr);
252
4.42k
  if (!PyObject_HasAttr(m_py_obj, py_attr.get()))
253
1.01k
    return PythonObject();
254
255
3.40k
  return PythonObject(PyRefType::Owned,
256
3.40k
                      PyObject_GetAttr(m_py_obj, py_attr.get()));
257
4.42k
}
258
259
1.72k
StructuredData::ObjectSP PythonObject::CreateStructuredObject() const {
260
1.72k
  switch (GetObjectType()) {
261
178
  case PyObjectType::Dictionary:
262
178
    return PythonDictionary(PyRefType::Borrowed, m_py_obj)
263
178
        .CreateStructuredDictionary();
264
12
  case PyObjectType::Boolean:
265
12
    return PythonBoolean(PyRefType::Borrowed, m_py_obj)
266
12
        .CreateStructuredBoolean();
267
780
  case PyObjectType::Integer:
268
780
    return PythonInteger(PyRefType::Borrowed, m_py_obj)
269
780
        .CreateStructuredInteger();
270
14
  case PyObjectType::List:
271
14
    return PythonList(PyRefType::Borrowed, m_py_obj).CreateStructuredArray();
272
736
  case PyObjectType::String:
273
736
    return PythonString(PyRefType::Borrowed, m_py_obj).CreateStructuredString();
274
6
  case PyObjectType::Bytes:
275
6
    return PythonBytes(PyRefType::Borrowed, m_py_obj).CreateStructuredString();
276
0
  case PyObjectType::ByteArray:
277
0
    return PythonByteArray(PyRefType::Borrowed, m_py_obj)
278
0
        .CreateStructuredString();
279
0
  case PyObjectType::None:
280
0
    return StructuredData::ObjectSP();
281
0
  default:
282
0
    return StructuredData::ObjectSP(new StructuredPythonObject(m_py_obj));
283
1.72k
  }
284
1.72k
}
285
286
// PythonString
287
288
0
PythonBytes::PythonBytes(llvm::ArrayRef<uint8_t> bytes) { SetBytes(bytes); }
289
290
16
PythonBytes::PythonBytes(const uint8_t *bytes, size_t length) {
291
16
  SetBytes(llvm::ArrayRef<uint8_t>(bytes, length));
292
16
}
293
294
846
bool PythonBytes::Check(PyObject *py_obj) {
295
846
  if (!py_obj)
296
0
    return false;
297
846
  return PyBytes_Check(py_obj);
298
846
}
299
300
7
llvm::ArrayRef<uint8_t> PythonBytes::GetBytes() const {
301
7
  if (!IsValid())
302
0
    return llvm::ArrayRef<uint8_t>();
303
304
7
  Py_ssize_t size;
305
7
  char *c;
306
307
7
  PyBytes_AsStringAndSize(m_py_obj, &c, &size);
308
7
  return llvm::ArrayRef<uint8_t>(reinterpret_cast<uint8_t *>(c), size);
309
7
}
310
311
6
size_t PythonBytes::GetSize() const {
312
6
  if (!IsValid())
313
0
    return 0;
314
6
  return PyBytes_Size(m_py_obj);
315
6
}
316
317
16
void PythonBytes::SetBytes(llvm::ArrayRef<uint8_t> bytes) {
318
16
  const char *data = reinterpret_cast<const char *>(bytes.data());
319
16
  *this = Take<PythonBytes>(PyBytes_FromStringAndSize(data, bytes.size()));
320
16
}
321
322
9
StructuredData::StringSP PythonBytes::CreateStructuredString() const {
323
9
  StructuredData::StringSP result(new StructuredData::String);
324
9
  Py_ssize_t size;
325
9
  char *c;
326
9
  PyBytes_AsStringAndSize(m_py_obj, &c, &size);
327
9
  result->SetValue(std::string(c, size));
328
9
  return result;
329
9
}
330
331
PythonByteArray::PythonByteArray(llvm::ArrayRef<uint8_t> bytes)
332
0
    : PythonByteArray(bytes.data(), bytes.size()) {}
Unexecuted instantiation: lldb_private::python::PythonByteArray::PythonByteArray(llvm::ArrayRef<unsigned char>)
Unexecuted instantiation: lldb_private::python::PythonByteArray::PythonByteArray(llvm::ArrayRef<unsigned char>)
333
334
0
PythonByteArray::PythonByteArray(const uint8_t *bytes, size_t length) {
335
0
  const char *str = reinterpret_cast<const char *>(bytes);
336
0
  *this = Take<PythonByteArray>(PyByteArray_FromStringAndSize(str, length));
337
0
}
338
339
822
bool PythonByteArray::Check(PyObject *py_obj) {
340
822
  if (!py_obj)
341
0
    return false;
342
822
  return PyByteArray_Check(py_obj);
343
822
}
344
345
8
llvm::ArrayRef<uint8_t> PythonByteArray::GetBytes() const {
346
8
  if (!IsValid())
347
0
    return llvm::ArrayRef<uint8_t>();
348
349
8
  char *c = PyByteArray_AsString(m_py_obj);
350
8
  size_t size = GetSize();
351
8
  return llvm::ArrayRef<uint8_t>(reinterpret_cast<uint8_t *>(c), size);
352
8
}
353
354
15
size_t PythonByteArray::GetSize() const {
355
15
  if (!IsValid())
356
0
    return 0;
357
358
15
  return PyByteArray_Size(m_py_obj);
359
15
}
360
361
0
StructuredData::StringSP PythonByteArray::CreateStructuredString() const {
362
0
  StructuredData::StringSP result(new StructuredData::String);
363
0
  llvm::ArrayRef<uint8_t> bytes = GetBytes();
364
0
  const char *str = reinterpret_cast<const char *>(bytes.data());
365
0
  result->SetValue(std::string(str, bytes.size()));
366
0
  return result;
367
0
}
368
369
// PythonString
370
371
61.6k
Expected<PythonString> PythonString::FromUTF8(llvm::StringRef string) {
372
61.6k
#if PY_MAJOR_VERSION >= 3
373
61.6k
  PyObject *str = PyUnicode_FromStringAndSize(string.data(), string.size());
374
#else
375
  PyObject *str = PyString_FromStringAndSize(string.data(), string.size());
376
#endif
377
61.6k
  if (!str)
378
0
    return llvm::make_error<PythonException>();
379
61.6k
  return Take<PythonString>(str);
380
61.6k
}
381
382
61.4k
PythonString::PythonString(llvm::StringRef string) { SetString(string); }
383
384
65.9k
bool PythonString::Check(PyObject *py_obj) {
385
65.9k
  if (!py_obj)
386
0
    return false;
387
388
65.9k
  if (PyUnicode_Check(py_obj))
389
65.1k
    return true;
390
#if PY_MAJOR_VERSION < 3
391
  if (PyString_Check(py_obj))
392
    return true;
393
#endif
394
821
  return false;
395
65.9k
}
396
397
64.1k
void PythonString::Convert(PyRefType &type, PyObject *&py_obj) {
398
#if PY_MAJOR_VERSION < 3
399
  // In Python 2, Don't store PyUnicode objects directly, because we need
400
  // access to their underlying character buffers which Python 2 doesn't
401
  // provide.
402
  if (PyUnicode_Check(py_obj)) {
403
    PyObject *s = PyUnicode_AsUTF8String(py_obj);
404
    if (s == nullptr) {
405
      PyErr_Clear();
406
      if (type == PyRefType::Owned)
407
        Py_DECREF(py_obj);
408
      return;
409
    }
410
    if (type == PyRefType::Owned)
411
      Py_DECREF(py_obj);
412
    else
413
      type = PyRefType::Owned;
414
    py_obj = s;
415
  }
416
#endif
417
64.1k
}
418
419
2.51k
llvm::StringRef PythonString::GetString() const {
420
2.51k
  auto s = AsUTF8();
421
2.51k
  if (!s) {
422
0
    llvm::consumeError(s.takeError());
423
0
    return llvm::StringRef("");
424
0
  }
425
2.51k
  return s.get();
426
2.51k
}
427
428
2.52k
Expected<llvm::StringRef> PythonString::AsUTF8() const {
429
2.52k
  if (!IsValid())
430
0
    return nullDeref();
431
432
2.52k
  Py_ssize_t size;
433
2.52k
  const char *data;
434
435
2.52k
#if PY_MAJOR_VERSION >= 3
436
2.52k
  data = PyUnicode_AsUTF8AndSize(m_py_obj, &size);
437
#else
438
  char *c = NULL;
439
  int r = PyString_AsStringAndSize(m_py_obj, &c, &size);
440
  if (r < 0)
441
    c = NULL;
442
  data = c;
443
#endif
444
445
2.52k
  if (!data)
446
0
    return exception();
447
448
2.52k
  return llvm::StringRef(data, size);
449
2.52k
}
450
451
25
size_t PythonString::GetSize() const {
452
25
  if (IsValid()) {
453
25
#if PY_MAJOR_VERSION >= 3
454
25
#if PY_MINOR_VERSION >= 3
455
25
    return PyUnicode_GetLength(m_py_obj);
456
#else
457
    return PyUnicode_GetSize(m_py_obj);
458
#endif
459
#else
460
    return PyString_Size(m_py_obj);
461
#endif
462
25
  }
463
0
  return 0;
464
25
}
465
466
61.4k
void PythonString::SetString(llvm::StringRef string) {
467
61.4k
  auto s = FromUTF8(string);
468
61.4k
  if (!s) {
469
0
    llvm::consumeError(s.takeError());
470
0
    Reset();
471
61.4k
  } else {
472
61.4k
    *this = std::move(s.get());
473
61.4k
  }
474
61.4k
}
475
476
737
StructuredData::StringSP PythonString::CreateStructuredString() const {
477
737
  StructuredData::StringSP result(new StructuredData::String);
478
737
  result->SetValue(GetString());
479
737
  return result;
480
737
}
481
482
// PythonInteger
483
484
1.42k
PythonInteger::PythonInteger(int64_t value) { SetInteger(value); }
485
486
2.99k
bool PythonInteger::Check(PyObject *py_obj) {
487
2.99k
  if (!py_obj)
488
0
    return false;
489
490
2.99k
#if PY_MAJOR_VERSION >= 3
491
  // Python 3 does not have PyInt_Check.  There is only one type of integral
492
  // value, long.
493
2.99k
  return PyLong_Check(py_obj);
494
#else
495
  return PyLong_Check(py_obj) || PyInt_Check(py_obj);
496
#endif
497
2.99k
}
498
499
2.20k
void PythonInteger::Convert(PyRefType &type, PyObject *&py_obj) {
500
#if PY_MAJOR_VERSION < 3
501
  // Always store this as a PyLong, which makes interoperability between Python
502
  // 2.x and Python 3.x easier.  This is only necessary in 2.x, since 3.x
503
  // doesn't even have a PyInt.
504
  if (PyInt_Check(py_obj)) {
505
    // Since we converted the original object to a different type, the new
506
    // object is an owned object regardless of the ownership semantics
507
    // requested by the user.
508
    long long value = PyInt_AsLong(py_obj);
509
    PyObject *l = nullptr;
510
    if (!PyErr_Occurred())
511
      l = PyLong_FromLongLong(value);
512
    if (l == nullptr) {
513
      PyErr_Clear();
514
      if (type == PyRefType::Owned)
515
        Py_DECREF(py_obj);
516
      return;
517
    }
518
    if (type == PyRefType::Owned)
519
      Py_DECREF(py_obj);
520
    else
521
      type = PyRefType::Owned;
522
    py_obj = l;
523
  }
524
#endif
525
2.20k
}
526
527
1.42k
void PythonInteger::SetInteger(int64_t value) {
528
1.42k
  *this = Take<PythonInteger>(PyLong_FromLongLong(value));
529
1.42k
}
530
531
781
StructuredData::IntegerSP PythonInteger::CreateStructuredInteger() const {
532
781
  StructuredData::IntegerSP result(new StructuredData::Integer);
533
  // FIXME this is really not ideal.   Errors are silently converted to 0
534
  // and overflows are silently wrapped.   But we'd need larger changes
535
  // to StructuredData to fix it, so that's how it is for now.
536
781
  llvm::Expected<unsigned long long> value = AsModuloUnsignedLongLong();
537
781
  if (!value) {
538
0
    llvm::consumeError(value.takeError());
539
0
    result->SetValue(0);
540
781
  } else {
541
781
    result->SetValue(value.get());
542
781
  }
543
781
  return result;
544
781
}
545
546
// PythonBoolean
547
548
0
PythonBoolean::PythonBoolean(bool value) {
549
0
  SetValue(value);
550
0
}
551
552
820
bool PythonBoolean::Check(PyObject *py_obj) {
553
820
  return py_obj ? PyBool_Check(py_obj) : 
false0
;
554
820
}
555
556
15
bool PythonBoolean::GetValue() const {
557
15
  return m_py_obj ? PyObject_IsTrue(m_py_obj) : 
false0
;
558
15
}
559
560
0
void PythonBoolean::SetValue(bool value) {
561
0
  *this = Take<PythonBoolean>(PyBool_FromLong(value));
562
0
}
563
564
12
StructuredData::BooleanSP PythonBoolean::CreateStructuredBoolean() const {
565
12
  StructuredData::BooleanSP result(new StructuredData::Boolean);
566
12
  result->SetValue(GetValue());
567
12
  return result;
568
12
}
569
570
// PythonList
571
572
2
PythonList::PythonList(PyInitialValue value) {
573
2
  if (value == PyInitialValue::Empty)
574
2
    *this = Take<PythonList>(PyList_New(0));
575
2
}
576
577
0
PythonList::PythonList(int list_size) {
578
0
  *this = Take<PythonList>(PyList_New(list_size));
579
0
}
580
581
3.53k
bool PythonList::Check(PyObject *py_obj) {
582
3.53k
  if (!py_obj)
583
0
    return false;
584
3.53k
  return PyList_Check(py_obj);
585
3.53k
}
586
587
365
uint32_t PythonList::GetSize() const {
588
365
  if (IsValid())
589
365
    return PyList_GET_SIZE(m_py_obj);
590
0
  return 0;
591
365
}
592
593
1.83k
PythonObject PythonList::GetItemAtIndex(uint32_t index) const {
594
1.83k
  if (IsValid())
595
1.83k
    return PythonObject(PyRefType::Borrowed, PyList_GetItem(m_py_obj, index));
596
0
  return PythonObject();
597
1.83k
}
598
599
2
void PythonList::SetItemAtIndex(uint32_t index, const PythonObject &object) {
600
2
  if (IsAllocated() && object.IsValid()) {
601
    // PyList_SetItem is documented to "steal" a reference, so we need to
602
    // convert it to an owned reference by incrementing it.
603
2
    Py_INCREF(object.get());
604
2
    PyList_SetItem(m_py_obj, index, object.get());
605
2
  }
606
2
}
607
608
4
void PythonList::AppendItem(const PythonObject &object) {
609
4
  if (IsAllocated() && object.IsValid()) {
610
    // `PyList_Append` does *not* steal a reference, so do not call `Py_INCREF`
611
    // here like we do with `PyList_SetItem`.
612
4
    PyList_Append(m_py_obj, object.get());
613
4
  }
614
4
}
615
616
31
StructuredData::ArraySP PythonList::CreateStructuredArray() const {
617
31
  StructuredData::ArraySP result(new StructuredData::Array);
618
31
  uint32_t count = GetSize();
619
211
  for (uint32_t i = 0; i < count; 
++i180
) {
620
180
    PythonObject obj = GetItemAtIndex(i);
621
180
    result->AddItem(obj.CreateStructuredObject());
622
180
  }
623
31
  return result;
624
31
}
625
626
// PythonTuple
627
628
1
PythonTuple::PythonTuple(PyInitialValue value) {
629
1
  if (value == PyInitialValue::Empty)
630
1
    *this = Take<PythonTuple>(PyTuple_New(0));
631
1
}
632
633
2
PythonTuple::PythonTuple(int tuple_size) {
634
2
  *this = Take<PythonTuple>(PyTuple_New(tuple_size));
635
2
}
636
637
2.36k
PythonTuple::PythonTuple(std::initializer_list<PythonObject> objects) {
638
2.36k
  m_py_obj = PyTuple_New(objects.size());
639
640
2.36k
  uint32_t idx = 0;
641
3.03k
  for (auto object : objects) {
642
3.03k
    if (object.IsValid())
643
3.03k
      SetItemAtIndex(idx, object);
644
3.03k
    idx++;
645
3.03k
  }
646
2.36k
}
647
648
2
PythonTuple::PythonTuple(std::initializer_list<PyObject *> objects) {
649
2
  m_py_obj = PyTuple_New(objects.size());
650
651
2
  uint32_t idx = 0;
652
5
  for (auto py_object : objects) {
653
5
    PythonObject object(PyRefType::Borrowed, py_object);
654
5
    if (object.IsValid())
655
5
      SetItemAtIndex(idx, object);
656
5
    idx++;
657
5
  }
658
2
}
659
660
1.72k
bool PythonTuple::Check(PyObject *py_obj) {
661
1.72k
  if (!py_obj)
662
0
    return false;
663
1.72k
  return PyTuple_Check(py_obj);
664
1.72k
}
665
666
6
uint32_t PythonTuple::GetSize() const {
667
6
  if (IsValid())
668
6
    return PyTuple_GET_SIZE(m_py_obj);
669
0
  return 0;
670
6
}
671
672
11
PythonObject PythonTuple::GetItemAtIndex(uint32_t index) const {
673
11
  if (IsValid())
674
11
    return PythonObject(PyRefType::Borrowed, PyTuple_GetItem(m_py_obj, index));
675
0
  return PythonObject();
676
11
}
677
678
3.04k
void PythonTuple::SetItemAtIndex(uint32_t index, const PythonObject &object) {
679
3.04k
  if (IsAllocated() && object.IsValid()) {
680
    // PyTuple_SetItem is documented to "steal" a reference, so we need to
681
    // convert it to an owned reference by incrementing it.
682
3.04k
    Py_INCREF(object.get());
683
3.04k
    PyTuple_SetItem(m_py_obj, index, object.get());
684
3.04k
  }
685
3.04k
}
686
687
1
StructuredData::ArraySP PythonTuple::CreateStructuredArray() const {
688
1
  StructuredData::ArraySP result(new StructuredData::Array);
689
1
  uint32_t count = GetSize();
690
3
  for (uint32_t i = 0; i < count; 
++i2
) {
691
2
    PythonObject obj = GetItemAtIndex(i);
692
2
    result->AddItem(obj.CreateStructuredObject());
693
2
  }
694
1
  return result;
695
1
}
696
697
// PythonDictionary
698
699
2.92k
PythonDictionary::PythonDictionary(PyInitialValue value) {
700
2.92k
  if (value == PyInitialValue::Empty)
701
58
    *this = Take<PythonDictionary>(PyDict_New());
702
2.92k
}
703
704
13.6k
bool PythonDictionary::Check(PyObject *py_obj) {
705
13.6k
  if (!py_obj)
706
0
    return false;
707
708
13.6k
  return PyDict_Check(py_obj);
709
13.6k
}
710
711
2
uint32_t PythonDictionary::GetSize() const {
712
2
  if (IsValid())
713
2
    return PyDict_Size(m_py_obj);
714
0
  return 0;
715
2
}
716
717
193
PythonList PythonDictionary::GetKeys() const {
718
193
  if (IsValid())
719
193
    return PythonList(PyRefType::Owned, PyDict_Keys(m_py_obj));
720
0
  return PythonList(PyInitialValue::Invalid);
721
193
}
722
723
20.6k
PythonObject PythonDictionary::GetItemForKey(const PythonObject &key) const {
724
20.6k
  auto item = GetItem(key);
725
20.6k
  if (!item) {
726
110
    llvm::consumeError(item.takeError());
727
110
    return PythonObject();
728
110
  }
729
20.5k
  return std::move(item.get());
730
20.6k
}
731
732
Expected<PythonObject>
733
20.6k
PythonDictionary::GetItem(const PythonObject &key) const {
734
20.6k
  if (!IsValid())
735
0
    return nullDeref();
736
20.6k
#if PY_MAJOR_VERSION >= 3
737
20.6k
  PyObject *o = PyDict_GetItemWithError(m_py_obj, key.get());
738
20.6k
  if (PyErr_Occurred())
739
0
    return exception();
740
#else
741
  PyObject *o = PyDict_GetItem(m_py_obj, key.get());
742
#endif
743
20.6k
  if (!o)
744
110
    return keyError();
745
20.5k
  return Retain<PythonObject>(o);
746
20.6k
}
747
748
1.31k
Expected<PythonObject> PythonDictionary::GetItem(const Twine &key) const {
749
1.31k
  if (!IsValid())
750
0
    return nullDeref();
751
1.31k
  PyObject *o = PyDict_GetItemString(m_py_obj, NullTerminated(key));
752
1.31k
  if (PyErr_Occurred())
753
0
    return exception();
754
1.31k
  if (!o)
755
0
    return keyError();
756
1.31k
  return Retain<PythonObject>(o);
757
1.31k
}
758
759
Error PythonDictionary::SetItem(const PythonObject &key,
760
37.0k
                                const PythonObject &value) const {
761
37.0k
  if (!IsValid() || !value.IsValid())
762
0
    return nullDeref();
763
37.0k
  int r = PyDict_SetItem(m_py_obj, key.get(), value.get());
764
37.0k
  if (r < 0)
765
0
    return exception();
766
37.0k
  return Error::success();
767
37.0k
}
768
769
Error PythonDictionary::SetItem(const Twine &key,
770
52
                                const PythonObject &value) const {
771
52
  if (!IsValid() || !value.IsValid())
772
0
    return nullDeref();
773
52
  int r = PyDict_SetItemString(m_py_obj, NullTerminated(key), value.get());
774
52
  if (r < 0)
775
0
    return exception();
776
52
  return Error::success();
777
52
}
778
779
void PythonDictionary::SetItemForKey(const PythonObject &key,
780
37.0k
                                     const PythonObject &value) {
781
37.0k
  Error error = SetItem(key, value);
782
37.0k
  if (error)
783
0
    llvm::consumeError(std::move(error));
784
37.0k
}
785
786
StructuredData::DictionarySP
787
193
PythonDictionary::CreateStructuredDictionary() const {
788
193
  StructuredData::DictionarySP result(new StructuredData::Dictionary);
789
193
  PythonList keys(GetKeys());
790
193
  uint32_t num_keys = keys.GetSize();
791
1.62k
  for (uint32_t i = 0; i < num_keys; 
++i1.43k
) {
792
1.43k
    PythonObject key = keys.GetItemAtIndex(i);
793
1.43k
    PythonObject value = GetItemForKey(key);
794
1.43k
    StructuredData::ObjectSP structured_value = value.CreateStructuredObject();
795
1.43k
    result->AddItem(key.Str().GetString(), structured_value);
796
1.43k
  }
797
193
  return result;
798
193
}
799
800
91
PythonModule PythonModule::BuiltinsModule() {
801
91
#if PY_MAJOR_VERSION >= 3
802
91
  return AddModule("builtins");
803
#else
804
  return AddModule("__builtin__");
805
#endif
806
91
}
807
808
484
PythonModule PythonModule::MainModule() { return AddModule("__main__"); }
809
810
575
PythonModule PythonModule::AddModule(llvm::StringRef module) {
811
575
  std::string str = module.str();
812
575
  return PythonModule(PyRefType::Borrowed, PyImport_AddModule(str.c_str()));
813
575
}
814
815
21.4k
Expected<PythonModule> PythonModule::Import(const Twine &name) {
816
21.4k
  PyObject *mod = PyImport_ImportModule(NullTerminated(name));
817
21.4k
  if (!mod)
818
0
    return exception();
819
21.4k
  return Take<PythonModule>(mod);
820
21.4k
}
821
822
18.9k
Expected<PythonObject> PythonModule::Get(const Twine &name) {
823
18.9k
  if (!IsValid())
824
0
    return nullDeref();
825
18.9k
  PyObject *dict = PyModule_GetDict(m_py_obj);
826
18.9k
  if (!dict)
827
0
    return exception();
828
18.9k
  PyObject *item = PyDict_GetItemString(dict, NullTerminated(name));
829
18.9k
  if (!item)
830
0
    return exception();
831
18.9k
  return Retain<PythonObject>(item);
832
18.9k
}
833
834
23.7k
bool PythonModule::Check(PyObject *py_obj) {
835
23.7k
  if (!py_obj)
836
0
    return false;
837
838
23.7k
  return PyModule_Check(py_obj);
839
23.7k
}
840
841
6.69k
PythonDictionary PythonModule::GetDictionary() const {
842
6.69k
  if (!IsValid())
843
0
    return PythonDictionary();
844
6.69k
  return Retain<PythonDictionary>(PyModule_GetDict(m_py_obj));
845
6.69k
}
846
847
7.02k
bool PythonCallable::Check(PyObject *py_obj) {
848
7.02k
  if (!py_obj)
849
1.02k
    return false;
850
851
6.00k
  return PyCallable_Check(py_obj);
852
7.02k
}
853
854
#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3
855
static const char get_arg_info_script[] = R"(
856
from inspect import signature, Parameter, ismethod
857
from collections import namedtuple
858
ArgInfo = namedtuple('ArgInfo', ['count', 'has_varargs'])
859
def main(f):
860
    count = 0
861
    varargs = False
862
    for parameter in signature(f).parameters.values():
863
        kind = parameter.kind
864
        if kind in (Parameter.POSITIONAL_ONLY,
865
                    Parameter.POSITIONAL_OR_KEYWORD):
866
            count += 1
867
        elif kind == Parameter.VAR_POSITIONAL:
868
            varargs = True
869
        elif kind in (Parameter.KEYWORD_ONLY,
870
                      Parameter.VAR_KEYWORD):
871
            pass
872
        else:
873
            raise Exception(f'unknown parameter kind: {kind}')
874
    return ArgInfo(count, varargs)
875
)";
876
#endif
877
878
397
Expected<PythonCallable::ArgInfo> PythonCallable::GetArgInfo() const {
879
397
  ArgInfo result = {};
880
397
  if (!IsValid())
881
0
    return nullDeref();
882
883
397
#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3
884
885
  // no need to synchronize access to this global, we already have the GIL
886
397
  static PythonScript get_arg_info(get_arg_info_script);
887
397
  Expected<PythonObject> pyarginfo = get_arg_info(*this);
888
397
  if (!pyarginfo)
889
0
    return pyarginfo.takeError();
890
397
  long long count =
891
397
      cantFail(As<long long>(pyarginfo.get().GetAttribute("count")));
892
397
  bool has_varargs =
893
397
      cantFail(As<bool>(pyarginfo.get().GetAttribute("has_varargs")));
894
397
  result.max_positional_args = has_varargs ? 
ArgInfo::UNBOUNDED4
:
count393
;
895
896
#else
897
  PyObject *py_func_obj;
898
  bool is_bound_method = false;
899
  bool is_class = false;
900
901
  if (PyType_Check(m_py_obj) || PyClass_Check(m_py_obj)) {
902
    auto init = GetAttribute("__init__");
903
    if (!init)
904
      return init.takeError();
905
    py_func_obj = init.get().get();
906
    is_class = true;
907
  } else {
908
    py_func_obj = m_py_obj;
909
  }
910
911
  if (PyMethod_Check(py_func_obj)) {
912
    py_func_obj = PyMethod_GET_FUNCTION(py_func_obj);
913
    PythonObject im_self = GetAttributeValue("im_self");
914
    if (im_self.IsValid() && !im_self.IsNone())
915
      is_bound_method = true;
916
  } else {
917
    // see if this is a callable object with an __call__ method
918
    if (!PyFunction_Check(py_func_obj)) {
919
      PythonObject __call__ = GetAttributeValue("__call__");
920
      if (__call__.IsValid()) {
921
        auto __callable__ = __call__.AsType<PythonCallable>();
922
        if (__callable__.IsValid()) {
923
          py_func_obj = PyMethod_GET_FUNCTION(__callable__.get());
924
          PythonObject im_self = __callable__.GetAttributeValue("im_self");
925
          if (im_self.IsValid() && !im_self.IsNone())
926
            is_bound_method = true;
927
        }
928
      }
929
    }
930
  }
931
932
  if (!py_func_obj)
933
    return result;
934
935
  PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(py_func_obj);
936
  if (!code)
937
    return result;
938
939
  auto count = code->co_argcount;
940
  bool has_varargs = !!(code->co_flags & CO_VARARGS);
941
  result.max_positional_args =
942
      has_varargs ? ArgInfo::UNBOUNDED
943
                  : (count - (int)is_bound_method) - (int)is_class;
944
945
#endif
946
947
397
  return result;
948
397
}
949
950
constexpr unsigned
951
    PythonCallable::ArgInfo::UNBOUNDED; // FIXME delete after c++17
952
953
433
PythonObject PythonCallable::operator()() {
954
433
  return PythonObject(PyRefType::Owned, PyObject_CallObject(m_py_obj, nullptr));
955
433
}
956
957
PythonObject PythonCallable::
958
0
operator()(std::initializer_list<PyObject *> args) {
959
0
  PythonTuple arg_tuple(args);
960
0
  return PythonObject(PyRefType::Owned,
961
0
                      PyObject_CallObject(m_py_obj, arg_tuple.get()));
962
0
}
963
964
PythonObject PythonCallable::
965
2.36k
operator()(std::initializer_list<PythonObject> args) {
966
2.36k
  PythonTuple arg_tuple(args);
967
2.36k
  return PythonObject(PyRefType::Owned,
968
2.36k
                      PyObject_CallObject(m_py_obj, arg_tuple.get()));
969
2.36k
}
970
971
18.8k
bool PythonFile::Check(PyObject *py_obj) {
972
18.8k
  if (!py_obj)
973
0
    return false;
974
#if PY_MAJOR_VERSION < 3
975
  return PyFile_Check(py_obj);
976
#else
977
  // In Python 3, there is no `PyFile_Check`, and in fact PyFile is not even a
978
  // first-class object type anymore.  `PyFile_FromFd` is just a thin wrapper
979
  // over `io.open()`, which returns some object derived from `io.IOBase`. As a
980
  // result, the only way to detect a file in Python 3 is to check whether it
981
  // inherits from `io.IOBase`.
982
18.8k
  auto io_module = PythonModule::Import("io");
983
18.8k
  if (!io_module) {
984
0
    llvm::consumeError(io_module.takeError());
985
0
    return false;
986
0
  }
987
18.8k
  auto iobase = io_module.get().Get("IOBase");
988
18.8k
  if (!iobase) {
989
0
    llvm::consumeError(iobase.takeError());
990
0
    return false;
991
0
  }
992
18.8k
  int r = PyObject_IsInstance(py_obj, iobase.get().get());
993
18.8k
  if (r < 0) {
994
0
    llvm::consumeError(exception()); // clear the exception and log it.
995
0
    return false;
996
0
  }
997
18.8k
  return !!r;
998
18.8k
#endif
999
18.8k
}
1000
1001
17
const char *PythonException::toCString() const {
1002
17
  if (!m_repr_bytes)
1003
0
    return "unknown exception";
1004
17
  return PyBytes_AS_STRING(m_repr_bytes);
1005
17
}
1006
1007
23
PythonException::PythonException(const char *caller) {
1008
23
  assert(PyErr_Occurred());
1009
0
  m_exception_type = m_exception = m_traceback = m_repr_bytes = NULL;
1010
23
  PyErr_Fetch(&m_exception_type, &m_exception, &m_traceback);
1011
23
  PyErr_NormalizeException(&m_exception_type, &m_exception, &m_traceback);
1012
23
  PyErr_Clear();
1013
23
  if (m_exception) {
1014
23
    PyObject *repr = PyObject_Repr(m_exception);
1015
23
    if (repr) {
1016
23
      m_repr_bytes = PyUnicode_AsEncodedString(repr, "utf-8", nullptr);
1017
23
      if (!m_repr_bytes) {
1018
0
        PyErr_Clear();
1019
0
      }
1020
23
      Py_XDECREF(repr);
1021
23
    } else {
1022
0
      PyErr_Clear();
1023
0
    }
1024
23
  }
1025
23
  Log *log = GetLogIfAllCategoriesSet(LIBLLDB_LOG_SCRIPT);
1026
23
  if (caller)
1027
0
    LLDB_LOGF(log, "%s failed with exception: %s", caller, toCString());
1028
23
  else
1029
23
    LLDB_LOGF(log, "python exception: %s", toCString());
1030
23
}
1031
5
void PythonException::Restore() {
1032
5
  if (m_exception_type && m_exception) {
1033
5
    PyErr_Restore(m_exception_type, m_exception, m_traceback);
1034
5
  } else {
1035
0
    PyErr_SetString(PyExc_Exception, toCString());
1036
0
  }
1037
5
  m_exception_type = m_exception = m_traceback = NULL;
1038
5
}
1039
1040
23
PythonException::~PythonException() {
1041
23
  Py_XDECREF(m_exception_type);
1042
23
  Py_XDECREF(m_exception);
1043
23
  Py_XDECREF(m_traceback);
1044
23
  Py_XDECREF(m_repr_bytes);
1045
23
}
1046
1047
10
void PythonException::log(llvm::raw_ostream &OS) const { OS << toCString(); }
1048
1049
0
std::error_code PythonException::convertToErrorCode() const {
1050
0
  return llvm::inconvertibleErrorCode();
1051
0
}
1052
1053
3
bool PythonException::Matches(PyObject *exc) const {
1054
3
  return PyErr_GivenExceptionMatches(m_exception_type, exc);
1055
3
}
1056
1057
const char read_exception_script[] = R"(
1058
import sys
1059
from traceback import print_exception
1060
if sys.version_info.major < 3:
1061
  from StringIO import StringIO
1062
else:
1063
  from io import StringIO
1064
def main(exc_type, exc_value, tb):
1065
  f = StringIO()
1066
  print_exception(exc_type, exc_value, tb, file=f)
1067
  return f.getvalue()
1068
)";
1069
1070
8
std::string PythonException::ReadBacktrace() const {
1071
1072
8
  if (!m_traceback)
1073
1
    return toCString();
1074
1075
  // no need to synchronize access to this global, we already have the GIL
1076
7
  static PythonScript read_exception(read_exception_script);
1077
1078
7
  Expected<std::string> backtrace = As<std::string>(
1079
7
      read_exception(m_exception_type, m_exception, m_traceback));
1080
1081
7
  if (!backtrace) {
1082
0
    std::string message =
1083
0
        std::string(toCString()) + "\n" +
1084
0
        "Traceback unavailable, an error occurred while reading it:\n";
1085
0
    return (message + llvm::toString(backtrace.takeError()));
1086
0
  }
1087
1088
7
  return std::move(backtrace.get());
1089
7
}
1090
1091
char PythonException::ID = 0;
1092
1093
llvm::Expected<File::OpenOptions>
1094
167
GetOptionsForPyObject(const PythonObject &obj) {
1095
167
#if PY_MAJOR_VERSION >= 3
1096
167
  auto options = File::OpenOptions(0);
1097
167
  auto readable = As<bool>(obj.CallMethod("readable"));
1098
167
  if (!readable)
1099
0
    return readable.takeError();
1100
167
  auto writable = As<bool>(obj.CallMethod("writable"));
1101
167
  if (!writable)
1102
1
    return writable.takeError();
1103
166
  if (readable.get() && 
writable.get()66
)
1104
4
    options |= File::eOpenOptionReadWrite;
1105
162
  else if (writable.get())
1106
100
    options |= File::eOpenOptionWriteOnly;
1107
62
  else if (readable.get())
1108
62
    options |= File::eOpenOptionReadOnly;
1109
166
  return options;
1110
#else
1111
  PythonString py_mode = obj.GetAttributeValue("mode").AsType<PythonString>();
1112
  return File::GetOptionsFromMode(py_mode.GetString());
1113
#endif
1114
167
}
1115
1116
// Base class template for python files.   All it knows how to do
1117
// is hold a reference to the python object and close or flush it
1118
// when the File is closed.
1119
namespace {
1120
template <typename Base> class OwnedPythonFile : public Base {
1121
public:
1122
  template <typename... Args>
1123
  OwnedPythonFile(const PythonFile &file, bool borrowed, Args... args)
1124
56
      : Base(args...), m_py_obj(file), m_borrowed(borrowed) {
1125
56
    assert(m_py_obj);
1126
56
  }
PythonDataObjects.cpp:(anonymous namespace)::OwnedPythonFile<lldb_private::NativeFile>::OwnedPythonFile<int, lldb_private::File::OpenOptions, bool>(lldb_private::python::PythonFile const&, bool, int, lldb_private::File::OpenOptions, bool)
Line
Count
Source
1124
24
      : Base(args...), m_py_obj(file), m_borrowed(borrowed) {
1125
24
    assert(m_py_obj);
1126
24
  }
PythonDataObjects.cpp:(anonymous namespace)::OwnedPythonFile<lldb_private::File>::OwnedPythonFile<>(lldb_private::python::PythonFile const&, bool)
Line
Count
Source
1124
32
      : Base(args...), m_py_obj(file), m_borrowed(borrowed) {
1125
32
    assert(m_py_obj);
1126
32
  }
1127
1128
56
  ~OwnedPythonFile() override {
1129
56
    assert(m_py_obj);
1130
0
    GIL takeGIL;
1131
56
    Close();
1132
    // we need to ensure the python object is released while we still
1133
    // hold the GIL
1134
56
    m_py_obj.Reset();
1135
56
  }
PythonDataObjects.cpp:(anonymous namespace)::OwnedPythonFile<lldb_private::NativeFile>::~OwnedPythonFile()
Line
Count
Source
1128
24
  ~OwnedPythonFile() override {
1129
24
    assert(m_py_obj);
1130
0
    GIL takeGIL;
1131
24
    Close();
1132
    // we need to ensure the python object is released while we still
1133
    // hold the GIL
1134
24
    m_py_obj.Reset();
1135
24
  }
PythonDataObjects.cpp:(anonymous namespace)::OwnedPythonFile<lldb_private::File>::~OwnedPythonFile()
Line
Count
Source
1128
32
  ~OwnedPythonFile() override {
1129
32
    assert(m_py_obj);
1130
0
    GIL takeGIL;
1131
32
    Close();
1132
    // we need to ensure the python object is released while we still
1133
    // hold the GIL
1134
32
    m_py_obj.Reset();
1135
32
  }
1136
1137
123
  bool IsPythonSideValid() const {
1138
123
    GIL takeGIL;
1139
123
    auto closed = As<bool>(m_py_obj.GetAttribute("closed"));
1140
123
    if (!closed) {
1141
0
      llvm::consumeError(closed.takeError());
1142
0
      return false;
1143
0
    }
1144
123
    return !closed.get();
1145
123
  }
PythonDataObjects.cpp:(anonymous namespace)::OwnedPythonFile<lldb_private::NativeFile>::IsPythonSideValid() const
Line
Count
Source
1137
62
  bool IsPythonSideValid() const {
1138
62
    GIL takeGIL;
1139
62
    auto closed = As<bool>(m_py_obj.GetAttribute("closed"));
1140
62
    if (!closed) {
1141
0
      llvm::consumeError(closed.takeError());
1142
0
      return false;
1143
0
    }
1144
62
    return !closed.get();
1145
62
  }
PythonDataObjects.cpp:(anonymous namespace)::OwnedPythonFile<lldb_private::File>::IsPythonSideValid() const
Line
Count
Source
1137
61
  bool IsPythonSideValid() const {
1138
61
    GIL takeGIL;
1139
61
    auto closed = As<bool>(m_py_obj.GetAttribute("closed"));
1140
61
    if (!closed) {
1141
0
      llvm::consumeError(closed.takeError());
1142
0
      return false;
1143
0
    }
1144
61
    return !closed.get();
1145
61
  }
1146
1147
62
  bool IsValid() const override {
1148
62
    return IsPythonSideValid() && Base::IsValid();
1149
62
  }
PythonDataObjects.cpp:(anonymous namespace)::OwnedPythonFile<lldb_private::NativeFile>::IsValid() const
Line
Count
Source
1147
62
  bool IsValid() const override {
1148
62
    return IsPythonSideValid() && Base::IsValid();
1149
62
  }
Unexecuted instantiation: PythonDataObjects.cpp:(anonymous namespace)::OwnedPythonFile<lldb_private::File>::IsValid() const
1150
1151
62
  Status Close() override {
1152
62
    assert(m_py_obj);
1153
0
    Status py_error, base_error;
1154
62
    GIL takeGIL;
1155
62
    if (!m_borrowed) {
1156
50
      auto r = m_py_obj.CallMethod("close");
1157
50
      if (!r)
1158
0
        py_error = Status(r.takeError());
1159
50
    }
1160
62
    base_error = Base::Close();
1161
62
    if (py_error.Fail())
1162
0
      return py_error;
1163
62
    return base_error;
1164
62
  };
PythonDataObjects.cpp:(anonymous namespace)::OwnedPythonFile<lldb_private::NativeFile>::Close()
Line
Count
Source
1151
30
  Status Close() override {
1152
30
    assert(m_py_obj);
1153
0
    Status py_error, base_error;
1154
30
    GIL takeGIL;
1155
30
    if (!m_borrowed) {
1156
30
      auto r = m_py_obj.CallMethod("close");
1157
30
      if (!r)
1158
0
        py_error = Status(r.takeError());
1159
30
    }
1160
30
    base_error = Base::Close();
1161
30
    if (py_error.Fail())
1162
0
      return py_error;
1163
30
    return base_error;
1164
30
  };
PythonDataObjects.cpp:(anonymous namespace)::OwnedPythonFile<lldb_private::File>::Close()
Line
Count
Source
1151
32
  Status Close() override {
1152
32
    assert(m_py_obj);
1153
0
    Status py_error, base_error;
1154
32
    GIL takeGIL;
1155
32
    if (!m_borrowed) {
1156
20
      auto r = m_py_obj.CallMethod("close");
1157
20
      if (!r)
1158
0
        py_error = Status(r.takeError());
1159
20
    }
1160
32
    base_error = Base::Close();
1161
32
    if (py_error.Fail())
1162
0
      return py_error;
1163
32
    return base_error;
1164
32
  };
1165
1166
7
  PyObject *GetPythonObject() const {
1167
7
    assert(m_py_obj.IsValid());
1168
0
    return m_py_obj.get();
1169
7
  }
PythonDataObjects.cpp:(anonymous namespace)::OwnedPythonFile<lldb_private::NativeFile>::GetPythonObject() const
Line
Count
Source
1166
1
  PyObject *GetPythonObject() const {
1167
1
    assert(m_py_obj.IsValid());
1168
0
    return m_py_obj.get();
1169
1
  }
PythonDataObjects.cpp:(anonymous namespace)::OwnedPythonFile<lldb_private::File>::GetPythonObject() const
Line
Count
Source
1166
6
  PyObject *GetPythonObject() const {
1167
6
    assert(m_py_obj.IsValid());
1168
0
    return m_py_obj.get();
1169
6
  }
1170
1171
  static bool classof(const File *file) = delete;
1172
1173
protected:
1174
  PythonFile m_py_obj;
1175
  bool m_borrowed;
1176
};
1177
} // namespace
1178
1179
// A SimplePythonFile is a OwnedPythonFile that just does all I/O as
1180
// a NativeFile
1181
namespace {
1182
class SimplePythonFile : public OwnedPythonFile<NativeFile> {
1183
public:
1184
  SimplePythonFile(const PythonFile &file, bool borrowed, int fd,
1185
                   File::OpenOptions options)
1186
24
      : OwnedPythonFile(file, borrowed, fd, options, false) {}
1187
1188
  static char ID;
1189
2
  bool isA(const void *classID) const override {
1190
2
    return classID == &ID || 
NativeFile::isA(classID)0
;
1191
2
  }
1192
18.5k
  static bool classof(const File *file) { return file->isA(&ID); }
1193
};
1194
char SimplePythonFile::ID = 0;
1195
} // namespace
1196
1197
#if PY_MAJOR_VERSION >= 3
1198
1199
namespace {
1200
class PythonBuffer {
1201
public:
1202
  PythonBuffer &operator=(const PythonBuffer &) = delete;
1203
  PythonBuffer(const PythonBuffer &) = delete;
1204
1205
  static Expected<PythonBuffer> Create(PythonObject &obj,
1206
3
                                       int flags = PyBUF_SIMPLE) {
1207
3
    Py_buffer py_buffer = {};
1208
3
    PyObject_GetBuffer(obj.get(), &py_buffer, flags);
1209
3
    if (!py_buffer.obj)
1210
0
      return llvm::make_error<PythonException>();
1211
3
    return PythonBuffer(py_buffer);
1212
3
  }
1213
1214
3
  PythonBuffer(PythonBuffer &&other) {
1215
3
    m_buffer = other.m_buffer;
1216
3
    other.m_buffer.obj = nullptr;
1217
3
  }
1218
1219
6
  ~PythonBuffer() {
1220
6
    if (m_buffer.obj)
1221
3
      PyBuffer_Release(&m_buffer);
1222
6
  }
1223
1224
9
  Py_buffer &get() { return m_buffer; }
1225
1226
private:
1227
  // takes ownership of the buffer.
1228
3
  PythonBuffer(const Py_buffer &py_buffer) : m_buffer(py_buffer) {}
1229
  Py_buffer m_buffer;
1230
};
1231
} // namespace
1232
1233
// Shared methods between TextPythonFile and BinaryPythonFile
1234
namespace {
1235
class PythonIOFile : public OwnedPythonFile<File> {
1236
public:
1237
  PythonIOFile(const PythonFile &file, bool borrowed)
1238
32
      : OwnedPythonFile(file, borrowed) {}
1239
1240
32
  ~PythonIOFile() override { Close(); }
1241
1242
61
  bool IsValid() const override { return IsPythonSideValid(); }
1243
1244
42
  Status Close() override {
1245
42
    assert(m_py_obj);
1246
0
    GIL takeGIL;
1247
42
    if (m_borrowed)
1248
15
      return Flush();
1249
27
    auto r = m_py_obj.CallMethod("close");
1250
27
    if (!r)
1251
3
      return Status(r.takeError());
1252
24
    return Status();
1253
27
  }
1254
1255
24
  Status Flush() override {
1256
24
    GIL takeGIL;
1257
24
    auto r = m_py_obj.CallMethod("flush");
1258
24
    if (!r)
1259
4
      return Status(r.takeError());
1260
20
    return Status();
1261
24
  }
1262
1263
0
  Expected<File::OpenOptions> GetOptions() const override {
1264
0
    GIL takeGIL;
1265
0
    return GetOptionsForPyObject(m_py_obj);
1266
0
  }
1267
1268
  static char ID;
1269
18
  bool isA(const void *classID) const override {
1270
18
    return classID == &ID || 
File::isA(classID)6
;
1271
18
  }
1272
18.5k
  static bool classof(const File *file) { return file->isA(&ID); }
1273
};
1274
char PythonIOFile::ID = 0;
1275
} // namespace
1276
1277
namespace {
1278
class BinaryPythonFile : public PythonIOFile {
1279
protected:
1280
  int m_descriptor;
1281
1282
public:
1283
  BinaryPythonFile(int fd, const PythonFile &file, bool borrowed)
1284
      : PythonIOFile(file, borrowed),
1285
        m_descriptor(File::DescriptorIsValid(fd) ? fd
1286
6
                                                 : File::kInvalidDescriptor) {}
1287
1288
2
  int GetDescriptor() const override { return m_descriptor; }
1289
1290
56
  Status Write(const void *buf, size_t &num_bytes) override {
1291
56
    GIL takeGIL;
1292
56
    PyObject *pybuffer_p = PyMemoryView_FromMemory(
1293
56
        const_cast<char *>((const char *)buf), num_bytes, PyBUF_READ);
1294
56
    if (!pybuffer_p)
1295
0
      return Status(llvm::make_error<PythonException>());
1296
56
    auto pybuffer = Take<PythonObject>(pybuffer_p);
1297
56
    num_bytes = 0;
1298
56
    auto bytes_written = As<long long>(m_py_obj.CallMethod("write", pybuffer));
1299
56
    if (!bytes_written)
1300
0
      return Status(bytes_written.takeError());
1301
56
    if (bytes_written.get() < 0)
1302
0
      return Status(".write() method returned a negative number!");
1303
56
    static_assert(sizeof(long long) >= sizeof(size_t), "overflow");
1304
56
    num_bytes = bytes_written.get();
1305
56
    return Status();
1306
56
  }
1307
1308
3
  Status Read(void *buf, size_t &num_bytes) override {
1309
3
    GIL takeGIL;
1310
3
    static_assert(sizeof(long long) >= sizeof(size_t), "overflow");
1311
3
    auto pybuffer_obj =
1312
3
        m_py_obj.CallMethod("read", (unsigned long long)num_bytes);
1313
3
    if (!pybuffer_obj)
1314
0
      return Status(pybuffer_obj.takeError());
1315
3
    num_bytes = 0;
1316
3
    if (pybuffer_obj.get().IsNone()) {
1317
      // EOF
1318
0
      num_bytes = 0;
1319
0
      return Status();
1320
0
    }
1321
3
    auto pybuffer = PythonBuffer::Create(pybuffer_obj.get());
1322
3
    if (!pybuffer)
1323
0
      return Status(pybuffer.takeError());
1324
3
    memcpy(buf, pybuffer.get().get().buf, pybuffer.get().get().len);
1325
3
    num_bytes = pybuffer.get().get().len;
1326
3
    return Status();
1327
3
  }
1328
};
1329
} // namespace
1330
1331
namespace {
1332
class TextPythonFile : public PythonIOFile {
1333
protected:
1334
  int m_descriptor;
1335
1336
public:
1337
  TextPythonFile(int fd, const PythonFile &file, bool borrowed)
1338
      : PythonIOFile(file, borrowed),
1339
        m_descriptor(File::DescriptorIsValid(fd) ? fd
1340
26
                                                 : File::kInvalidDescriptor) {}
1341
1342
2
  int GetDescriptor() const override { return m_descriptor; }
1343
1344
185
  Status Write(const void *buf, size_t &num_bytes) override {
1345
185
    GIL takeGIL;
1346
185
    auto pystring =
1347
185
        PythonString::FromUTF8(llvm::StringRef((const char *)buf, num_bytes));
1348
185
    if (!pystring)
1349
0
      return Status(pystring.takeError());
1350
185
    num_bytes = 0;
1351
185
    auto bytes_written =
1352
185
        As<long long>(m_py_obj.CallMethod("write", pystring.get()));
1353
185
    if (!bytes_written)
1354
2
      return Status(bytes_written.takeError());
1355
183
    if (bytes_written.get() < 0)
1356
0
      return Status(".write() method returned a negative number!");
1357
183
    static_assert(sizeof(long long) >= sizeof(size_t), "overflow");
1358
183
    num_bytes = bytes_written.get();
1359
183
    return Status();
1360
183
  }
1361
1362
5
  Status Read(void *buf, size_t &num_bytes) override {
1363
5
    GIL takeGIL;
1364
5
    size_t num_chars = num_bytes / 6;
1365
5
    size_t orig_num_bytes = num_bytes;
1366
5
    num_bytes = 0;
1367
5
    if (orig_num_bytes < 6) {
1368
1
      return Status("can't read less than 6 bytes from a utf8 text stream");
1369
1
    }
1370
4
    auto pystring = As<PythonString>(
1371
4
        m_py_obj.CallMethod("read", (unsigned long long)num_chars));
1372
4
    if (!pystring)
1373
1
      return Status(pystring.takeError());
1374
3
    if (pystring.get().IsNone()) {
1375
      // EOF
1376
0
      return Status();
1377
0
    }
1378
3
    auto stringref = pystring.get().AsUTF8();
1379
3
    if (!stringref)
1380
0
      return Status(stringref.takeError());
1381
3
    num_bytes = stringref.get().size();
1382
3
    memcpy(buf, stringref.get().begin(), num_bytes);
1383
3
    return Status();
1384
3
  }
1385
};
1386
} // namespace
1387
1388
#endif
1389
1390
195
llvm::Expected<FileSP> PythonFile::ConvertToFile(bool borrowed) {
1391
195
  if (!IsValid())
1392
0
    return llvm::createStringError(llvm::inconvertibleErrorCode(),
1393
0
                                   "invalid PythonFile");
1394
1395
195
  int fd = PyObject_AsFileDescriptor(m_py_obj);
1396
195
  if (fd < 0) {
1397
28
    PyErr_Clear();
1398
28
    return ConvertToFileForcingUseOfScriptingIOMethods(borrowed);
1399
28
  }
1400
167
  auto options = GetOptionsForPyObject(*this);
1401
167
  if (!options)
1402
1
    return options.takeError();
1403
1404
166
  File::OpenOptions rw =
1405
166
      options.get() & (File::eOpenOptionReadOnly | File::eOpenOptionWriteOnly |
1406
166
                       File::eOpenOptionReadWrite);
1407
166
  if (rw == File::eOpenOptionWriteOnly || 
rw == File::eOpenOptionReadWrite66
) {
1408
    // LLDB and python will not share I/O buffers.  We should probably
1409
    // flush the python buffers now.
1410
104
    auto r = CallMethod("flush");
1411
104
    if (!r)
1412
0
      return r.takeError();
1413
104
  }
1414
1415
166
  FileSP file_sp;
1416
166
  if (borrowed) {
1417
    // In this case we we don't need to retain the python
1418
    // object at all.
1419
142
    file_sp = std::make_shared<NativeFile>(fd, options.get(), false);
1420
142
  } else {
1421
24
    file_sp = std::static_pointer_cast<File>(
1422
24
        std::make_shared<SimplePythonFile>(*this, borrowed, fd, options.get()));
1423
24
  }
1424
166
  if (!file_sp->IsValid())
1425
0
    return llvm::createStringError(llvm::inconvertibleErrorCode(),
1426
0
                                   "invalid File");
1427
1428
166
  return file_sp;
1429
166
}
1430
1431
llvm::Expected<FileSP>
1432
32
PythonFile::ConvertToFileForcingUseOfScriptingIOMethods(bool borrowed) {
1433
1434
32
  assert(!PyErr_Occurred());
1435
1436
32
  if (!IsValid())
1437
0
    return llvm::createStringError(llvm::inconvertibleErrorCode(),
1438
0
                                   "invalid PythonFile");
1439
1440
#if PY_MAJOR_VERSION < 3
1441
1442
  return llvm::createStringError(llvm::inconvertibleErrorCode(),
1443
                                 "not supported on python 2");
1444
1445
#else
1446
1447
32
  int fd = PyObject_AsFileDescriptor(m_py_obj);
1448
32
  if (fd < 0) {
1449
28
    PyErr_Clear();
1450
28
    fd = File::kInvalidDescriptor;
1451
28
  }
1452
1453
32
  auto io_module = PythonModule::Import("io");
1454
32
  if (!io_module)
1455
0
    return io_module.takeError();
1456
32
  auto textIOBase = io_module.get().Get("TextIOBase");
1457
32
  if (!textIOBase)
1458
0
    return textIOBase.takeError();
1459
32
  auto rawIOBase = io_module.get().Get("RawIOBase");
1460
32
  if (!rawIOBase)
1461
0
    return rawIOBase.takeError();
1462
32
  auto bufferedIOBase = io_module.get().Get("BufferedIOBase");
1463
32
  if (!bufferedIOBase)
1464
0
    return bufferedIOBase.takeError();
1465
1466
32
  FileSP file_sp;
1467
1468
32
  auto isTextIO = IsInstance(textIOBase.get());
1469
32
  if (!isTextIO)
1470
0
    return isTextIO.takeError();
1471
32
  if (isTextIO.get())
1472
26
    file_sp = std::static_pointer_cast<File>(
1473
26
        std::make_shared<TextPythonFile>(fd, *this, borrowed));
1474
1475
32
  auto isRawIO = IsInstance(rawIOBase.get());
1476
32
  if (!isRawIO)
1477
0
    return isRawIO.takeError();
1478
32
  auto isBufferedIO = IsInstance(bufferedIOBase.get());
1479
32
  if (!isBufferedIO)
1480
0
    return isBufferedIO.takeError();
1481
1482
32
  if (isRawIO.get() || isBufferedIO.get()) {
1483
6
    file_sp = std::static_pointer_cast<File>(
1484
6
        std::make_shared<BinaryPythonFile>(fd, *this, borrowed));
1485
6
  }
1486
1487
32
  if (!file_sp)
1488
0
    return llvm::createStringError(llvm::inconvertibleErrorCode(),
1489
0
                                   "python file is neither text nor binary");
1490
1491
32
  if (!file_sp->IsValid())
1492
0
    return llvm::createStringError(llvm::inconvertibleErrorCode(),
1493
0
                                   "invalid File");
1494
1495
32
  return file_sp;
1496
1497
32
#endif
1498
32
}
1499
1500
18.5k
Expected<PythonFile> PythonFile::FromFile(File &file, const char *mode) {
1501
18.5k
  if (!file.IsValid())
1502
0
    return llvm::createStringError(llvm::inconvertibleErrorCode(),
1503
0
                                   "invalid file");
1504
1505
18.5k
  if (auto *simple = llvm::dyn_cast<SimplePythonFile>(&file))
1506
1
    return Retain<PythonFile>(simple->GetPythonObject());
1507
18.5k
#if PY_MAJOR_VERSION >= 3
1508
18.5k
  if (auto *pythonio = llvm::dyn_cast<PythonIOFile>(&file))
1509
6
    return Retain<PythonFile>(pythonio->GetPythonObject());
1510
18.5k
#endif
1511
1512
18.5k
  if (!mode) {
1513
18
    auto m = file.GetOpenMode();
1514
18
    if (!m)
1515
0
      return m.takeError();
1516
18
    mode = m.get();
1517
18
  }
1518
1519
18.5k
  PyObject *file_obj;
1520
18.5k
#if PY_MAJOR_VERSION >= 3
1521
18.5k
  file_obj = PyFile_FromFd(file.GetDescriptor(), nullptr, mode, -1, nullptr,
1522
18.5k
                           "ignore", nullptr, /*closefd=*/0);
1523
#else
1524
  // I'd like to pass ::fflush here if the file is writable,  so that
1525
  // when the python side destructs the file object it will be flushed.
1526
  // However, this would be dangerous.    It can cause fflush to be called
1527
  // after fclose if the python program keeps a reference to the file after
1528
  // the original lldb_private::File has been destructed.
1529
  //
1530
  // It's all well and good to ask a python program not to use a closed file
1531
  // but asking a python program to make sure objects get released in a
1532
  // particular order is not safe.
1533
  //
1534
  // The tradeoff here is that if a python 2 program wants to make sure this
1535
  // file gets flushed, they'll have to do it explicitly or wait untill the
1536
  // original lldb File itself gets flushed.
1537
  file_obj = PyFile_FromFile(file.GetStream(), py2_const_cast(""),
1538
                             py2_const_cast(mode), [](FILE *) { return 0; });
1539
#endif
1540
1541
18.5k
  if (!file_obj)
1542
0
    return exception();
1543
1544
18.5k
  return Take<PythonFile>(file_obj);
1545
18.5k
}
1546
1547
408
Error PythonScript::Init() {
1548
408
  if (function.IsValid())
1549
357
    return Error::success();
1550
1551
51
  PythonDictionary globals(PyInitialValue::Empty);
1552
51
  auto builtins = PythonModule::BuiltinsModule();
1553
51
  if (Error error = globals.SetItem("__builtins__", builtins))
1554
0
    return error;
1555
51
  PyObject *o =
1556
51
      PyRun_String(script, Py_file_input, globals.get(), globals.get());
1557
51
  if (!o)
1558
0
    return exception();
1559
51
  Take<PythonObject>(o);
1560
51
  auto f = As<PythonCallable>(globals.GetItem("main"));
1561
51
  if (!f)
1562
0
    return f.takeError();
1563
51
  function = std::move(f.get());
1564
1565
51
  return Error::success();
1566
51
}
1567
1568
llvm::Expected<PythonObject>
1569
python::runStringOneLine(const llvm::Twine &string,
1570
                         const PythonDictionary &globals,
1571
5.16k
                         const PythonDictionary &locals) {
1572
5.16k
  if (!globals.IsValid() || !locals.IsValid())
1573
0
    return nullDeref();
1574
1575
5.16k
  PyObject *code =
1576
5.16k
      Py_CompileString(NullTerminated(string), "<string>", Py_eval_input);
1577
5.16k
  if (!code) {
1578
1
    PyErr_Clear();
1579
1
    code =
1580
1
        Py_CompileString(NullTerminated(string), "<string>", Py_single_input);
1581
1
  }
1582
5.16k
  if (!code)
1583
0
    return exception();
1584
5.16k
  auto code_ref = Take<PythonObject>(code);
1585
1586
#if PY_MAJOR_VERSION < 3
1587
  PyObject *result =
1588
      PyEval_EvalCode((PyCodeObject *)code, globals.get(), locals.get());
1589
#else
1590
5.16k
  PyObject *result = PyEval_EvalCode(code, globals.get(), locals.get());
1591
5.16k
#endif
1592
1593
5.16k
  if (!result)
1594
3
    return exception();
1595
1596
5.16k
  return Take<PythonObject>(result);
1597
5.16k
}
1598
1599
llvm::Expected<PythonObject>
1600
python::runStringMultiLine(const llvm::Twine &string,
1601
                           const PythonDictionary &globals,
1602
264
                           const PythonDictionary &locals) {
1603
264
  if (!globals.IsValid() || !locals.IsValid())
1604
0
    return nullDeref();
1605
264
  PyObject *result = PyRun_String(NullTerminated(string), Py_file_input,
1606
264
                                  globals.get(), locals.get());
1607
264
  if (!result)
1608
5
    return exception();
1609
259
  return Take<PythonObject>(result);
1610
264
}
1611
1612
#endif