xref: /freebsd/contrib/llvm-project/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583)
1  //===-- PythonDataObjects.cpp ---------------------------------------------===//
2  //
3  // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4  // See https://llvm.org/LICENSE.txt for license information.
5  // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6  //
7  //===----------------------------------------------------------------------===//
8  
9  #include "lldb/Host/Config.h"
10  
11  #if LLDB_ENABLE_PYTHON
12  
13  #include "PythonDataObjects.h"
14  #include "ScriptInterpreterPython.h"
15  
16  #include "lldb/Host/File.h"
17  #include "lldb/Host/FileSystem.h"
18  #include "lldb/Interpreter/ScriptInterpreter.h"
19  #include "lldb/Utility/LLDBLog.h"
20  #include "lldb/Utility/Log.h"
21  #include "lldb/Utility/Stream.h"
22  
23  #include "llvm/Support/Casting.h"
24  #include "llvm/Support/ConvertUTF.h"
25  #include "llvm/Support/Errno.h"
26  
27  #include <cstdio>
28  #include <variant>
29  
30  using namespace lldb_private;
31  using namespace lldb;
32  using namespace lldb_private::python;
33  using llvm::cantFail;
34  using llvm::Error;
35  using llvm::Expected;
36  using llvm::Twine;
37  
As(Expected<PythonObject> && obj)38  template <> Expected<bool> python::As<bool>(Expected<PythonObject> &&obj) {
39    if (!obj)
40      return obj.takeError();
41    return obj.get().IsTrue();
42  }
43  
44  template <>
As(Expected<PythonObject> && obj)45  Expected<long long> python::As<long long>(Expected<PythonObject> &&obj) {
46    if (!obj)
47      return obj.takeError();
48    return obj->AsLongLong();
49  }
50  
51  template <>
52  Expected<unsigned long long>
As(Expected<PythonObject> && obj)53  python::As<unsigned long long>(Expected<PythonObject> &&obj) {
54    if (!obj)
55      return obj.takeError();
56    return obj->AsUnsignedLongLong();
57  }
58  
59  template <>
As(Expected<PythonObject> && obj)60  Expected<std::string> python::As<std::string>(Expected<PythonObject> &&obj) {
61    if (!obj)
62      return obj.takeError();
63    PyObject *str_obj = PyObject_Str(obj.get().get());
64    if (!str_obj)
65      return llvm::make_error<PythonException>();
66    auto str = Take<PythonString>(str_obj);
67    auto utf8 = str.AsUTF8();
68    if (!utf8)
69      return utf8.takeError();
70    return std::string(utf8.get());
71  }
72  
python_is_finalizing()73  static bool python_is_finalizing() {
74  #if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 13) || (PY_MAJOR_VERSION > 3)
75    return Py_IsFinalizing();
76  #elif PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 7
77    return _Py_Finalizing != nullptr;
78  #else
79    return _Py_IsFinalizing();
80  #endif
81  }
82  
Reset()83  void PythonObject::Reset() {
84    if (m_py_obj && Py_IsInitialized()) {
85      if (python_is_finalizing()) {
86        // Leak m_py_obj rather than crashing the process.
87        // https://docs.python.org/3/c-api/init.html#c.PyGILState_Ensure
88      } else {
89        PyGILState_STATE state = PyGILState_Ensure();
90        Py_DECREF(m_py_obj);
91        PyGILState_Release(state);
92      }
93    }
94    m_py_obj = nullptr;
95  }
96  
AsLongLong() const97  Expected<long long> PythonObject::AsLongLong() const {
98    if (!m_py_obj)
99      return nullDeref();
100    assert(!PyErr_Occurred());
101    long long r = PyLong_AsLongLong(m_py_obj);
102    if (PyErr_Occurred())
103      return exception();
104    return r;
105  }
106  
AsUnsignedLongLong() const107  Expected<unsigned long long> PythonObject::AsUnsignedLongLong() const {
108    if (!m_py_obj)
109      return nullDeref();
110    assert(!PyErr_Occurred());
111    long long r = PyLong_AsUnsignedLongLong(m_py_obj);
112    if (PyErr_Occurred())
113      return exception();
114    return r;
115  }
116  
117  // wraps on overflow, instead of raising an error.
AsModuloUnsignedLongLong() const118  Expected<unsigned long long> PythonObject::AsModuloUnsignedLongLong() const {
119    if (!m_py_obj)
120      return nullDeref();
121    assert(!PyErr_Occurred());
122    unsigned long long r = PyLong_AsUnsignedLongLongMask(m_py_obj);
123    // FIXME: We should fetch the exception message and hoist it.
124    if (PyErr_Occurred())
125      return exception();
126    return r;
127  }
128  
Serialize(llvm::json::OStream & s) const129  void StructuredPythonObject::Serialize(llvm::json::OStream &s) const {
130    s.value(llvm::formatv("Python Obj: {0:X}", GetValue()).str());
131  }
132  
133  // PythonObject
134  
Dump(Stream & strm) const135  void PythonObject::Dump(Stream &strm) const {
136    if (m_py_obj) {
137      FILE *file = llvm::sys::RetryAfterSignal(nullptr, ::tmpfile);
138      if (file) {
139        ::PyObject_Print(m_py_obj, file, 0);
140        const long length = ftell(file);
141        if (length) {
142          ::rewind(file);
143          std::vector<char> file_contents(length, '\0');
144          const size_t length_read =
145              ::fread(file_contents.data(), 1, file_contents.size(), file);
146          if (length_read > 0)
147            strm.Write(file_contents.data(), length_read);
148        }
149        ::fclose(file);
150      }
151    } else
152      strm.PutCString("NULL");
153  }
154  
GetObjectType() const155  PyObjectType PythonObject::GetObjectType() const {
156    if (!IsAllocated())
157      return PyObjectType::None;
158  
159    if (PythonModule::Check(m_py_obj))
160      return PyObjectType::Module;
161    if (PythonList::Check(m_py_obj))
162      return PyObjectType::List;
163    if (PythonTuple::Check(m_py_obj))
164      return PyObjectType::Tuple;
165    if (PythonDictionary::Check(m_py_obj))
166      return PyObjectType::Dictionary;
167    if (PythonString::Check(m_py_obj))
168      return PyObjectType::String;
169    if (PythonBytes::Check(m_py_obj))
170      return PyObjectType::Bytes;
171    if (PythonByteArray::Check(m_py_obj))
172      return PyObjectType::ByteArray;
173    if (PythonBoolean::Check(m_py_obj))
174      return PyObjectType::Boolean;
175    if (PythonInteger::Check(m_py_obj))
176      return PyObjectType::Integer;
177    if (PythonFile::Check(m_py_obj))
178      return PyObjectType::File;
179    if (PythonCallable::Check(m_py_obj))
180      return PyObjectType::Callable;
181    return PyObjectType::Unknown;
182  }
183  
Repr() const184  PythonString PythonObject::Repr() const {
185    if (!m_py_obj)
186      return PythonString();
187    PyObject *repr = PyObject_Repr(m_py_obj);
188    if (!repr)
189      return PythonString();
190    return PythonString(PyRefType::Owned, repr);
191  }
192  
Str() const193  PythonString PythonObject::Str() const {
194    if (!m_py_obj)
195      return PythonString();
196    PyObject *str = PyObject_Str(m_py_obj);
197    if (!str)
198      return PythonString();
199    return PythonString(PyRefType::Owned, str);
200  }
201  
202  PythonObject
ResolveNameWithDictionary(llvm::StringRef name,const PythonDictionary & dict)203  PythonObject::ResolveNameWithDictionary(llvm::StringRef name,
204                                          const PythonDictionary &dict) {
205    size_t dot_pos = name.find('.');
206    llvm::StringRef piece = name.substr(0, dot_pos);
207    PythonObject result = dict.GetItemForKey(PythonString(piece));
208    if (dot_pos == llvm::StringRef::npos) {
209      // There was no dot, we're done.
210      return result;
211    }
212  
213    // There was a dot.  The remaining portion of the name should be looked up in
214    // the context of the object that was found in the dictionary.
215    return result.ResolveName(name.substr(dot_pos + 1));
216  }
217  
ResolveName(llvm::StringRef name) const218  PythonObject PythonObject::ResolveName(llvm::StringRef name) const {
219    // Resolve the name in the context of the specified object.  If, for example,
220    // `this` refers to a PyModule, then this will look for `name` in this
221    // module.  If `this` refers to a PyType, then it will resolve `name` as an
222    // attribute of that type.  If `this` refers to an instance of an object,
223    // then it will resolve `name` as the value of the specified field.
224    //
225    // This function handles dotted names so that, for example, if `m_py_obj`
226    // refers to the `sys` module, and `name` == "path.append", then it will find
227    // the function `sys.path.append`.
228  
229    size_t dot_pos = name.find('.');
230    if (dot_pos == llvm::StringRef::npos) {
231      // No dots in the name, we should be able to find the value immediately as
232      // an attribute of `m_py_obj`.
233      return GetAttributeValue(name);
234    }
235  
236    // Look up the first piece of the name, and resolve the rest as a child of
237    // that.
238    PythonObject parent = ResolveName(name.substr(0, dot_pos));
239    if (!parent.IsAllocated())
240      return PythonObject();
241  
242    // Tail recursion.. should be optimized by the compiler
243    return parent.ResolveName(name.substr(dot_pos + 1));
244  }
245  
HasAttribute(llvm::StringRef attr) const246  bool PythonObject::HasAttribute(llvm::StringRef attr) const {
247    if (!IsValid())
248      return false;
249    PythonString py_attr(attr);
250    return !!PyObject_HasAttr(m_py_obj, py_attr.get());
251  }
252  
GetAttributeValue(llvm::StringRef attr) const253  PythonObject PythonObject::GetAttributeValue(llvm::StringRef attr) const {
254    if (!IsValid())
255      return PythonObject();
256  
257    PythonString py_attr(attr);
258    if (!PyObject_HasAttr(m_py_obj, py_attr.get()))
259      return PythonObject();
260  
261    return PythonObject(PyRefType::Owned,
262                        PyObject_GetAttr(m_py_obj, py_attr.get()));
263  }
264  
CreateStructuredObject() const265  StructuredData::ObjectSP PythonObject::CreateStructuredObject() const {
266    assert(PyGILState_Check());
267    switch (GetObjectType()) {
268    case PyObjectType::Dictionary:
269      return PythonDictionary(PyRefType::Borrowed, m_py_obj)
270          .CreateStructuredDictionary();
271    case PyObjectType::Boolean:
272      return PythonBoolean(PyRefType::Borrowed, m_py_obj)
273          .CreateStructuredBoolean();
274    case PyObjectType::Integer: {
275      StructuredData::IntegerSP int_sp =
276          PythonInteger(PyRefType::Borrowed, m_py_obj).CreateStructuredInteger();
277      if (std::holds_alternative<StructuredData::UnsignedIntegerSP>(int_sp))
278        return std::get<StructuredData::UnsignedIntegerSP>(int_sp);
279      if (std::holds_alternative<StructuredData::SignedIntegerSP>(int_sp))
280        return std::get<StructuredData::SignedIntegerSP>(int_sp);
281      return nullptr;
282    };
283    case PyObjectType::List:
284      return PythonList(PyRefType::Borrowed, m_py_obj).CreateStructuredArray();
285    case PyObjectType::String:
286      return PythonString(PyRefType::Borrowed, m_py_obj).CreateStructuredString();
287    case PyObjectType::Bytes:
288      return PythonBytes(PyRefType::Borrowed, m_py_obj).CreateStructuredString();
289    case PyObjectType::ByteArray:
290      return PythonByteArray(PyRefType::Borrowed, m_py_obj)
291          .CreateStructuredString();
292    case PyObjectType::None:
293      return StructuredData::ObjectSP();
294    default:
295      return StructuredData::ObjectSP(new StructuredPythonObject(
296          PythonObject(PyRefType::Borrowed, m_py_obj)));
297    }
298  }
299  
300  // PythonString
301  
PythonBytes(llvm::ArrayRef<uint8_t> bytes)302  PythonBytes::PythonBytes(llvm::ArrayRef<uint8_t> bytes) { SetBytes(bytes); }
303  
PythonBytes(const uint8_t * bytes,size_t length)304  PythonBytes::PythonBytes(const uint8_t *bytes, size_t length) {
305    SetBytes(llvm::ArrayRef<uint8_t>(bytes, length));
306  }
307  
Check(PyObject * py_obj)308  bool PythonBytes::Check(PyObject *py_obj) {
309    if (!py_obj)
310      return false;
311    return PyBytes_Check(py_obj);
312  }
313  
GetBytes() const314  llvm::ArrayRef<uint8_t> PythonBytes::GetBytes() const {
315    if (!IsValid())
316      return llvm::ArrayRef<uint8_t>();
317  
318    Py_ssize_t size;
319    char *c;
320  
321    PyBytes_AsStringAndSize(m_py_obj, &c, &size);
322    return llvm::ArrayRef<uint8_t>(reinterpret_cast<uint8_t *>(c), size);
323  }
324  
GetSize() const325  size_t PythonBytes::GetSize() const {
326    if (!IsValid())
327      return 0;
328    return PyBytes_Size(m_py_obj);
329  }
330  
SetBytes(llvm::ArrayRef<uint8_t> bytes)331  void PythonBytes::SetBytes(llvm::ArrayRef<uint8_t> bytes) {
332    const char *data = reinterpret_cast<const char *>(bytes.data());
333    *this = Take<PythonBytes>(PyBytes_FromStringAndSize(data, bytes.size()));
334  }
335  
CreateStructuredString() const336  StructuredData::StringSP PythonBytes::CreateStructuredString() const {
337    StructuredData::StringSP result(new StructuredData::String);
338    Py_ssize_t size;
339    char *c;
340    PyBytes_AsStringAndSize(m_py_obj, &c, &size);
341    result->SetValue(std::string(c, size));
342    return result;
343  }
344  
PythonByteArray(llvm::ArrayRef<uint8_t> bytes)345  PythonByteArray::PythonByteArray(llvm::ArrayRef<uint8_t> bytes)
346      : PythonByteArray(bytes.data(), bytes.size()) {}
347  
PythonByteArray(const uint8_t * bytes,size_t length)348  PythonByteArray::PythonByteArray(const uint8_t *bytes, size_t length) {
349    const char *str = reinterpret_cast<const char *>(bytes);
350    *this = Take<PythonByteArray>(PyByteArray_FromStringAndSize(str, length));
351  }
352  
Check(PyObject * py_obj)353  bool PythonByteArray::Check(PyObject *py_obj) {
354    if (!py_obj)
355      return false;
356    return PyByteArray_Check(py_obj);
357  }
358  
GetBytes() const359  llvm::ArrayRef<uint8_t> PythonByteArray::GetBytes() const {
360    if (!IsValid())
361      return llvm::ArrayRef<uint8_t>();
362  
363    char *c = PyByteArray_AsString(m_py_obj);
364    size_t size = GetSize();
365    return llvm::ArrayRef<uint8_t>(reinterpret_cast<uint8_t *>(c), size);
366  }
367  
GetSize() const368  size_t PythonByteArray::GetSize() const {
369    if (!IsValid())
370      return 0;
371  
372    return PyByteArray_Size(m_py_obj);
373  }
374  
CreateStructuredString() const375  StructuredData::StringSP PythonByteArray::CreateStructuredString() const {
376    StructuredData::StringSP result(new StructuredData::String);
377    llvm::ArrayRef<uint8_t> bytes = GetBytes();
378    const char *str = reinterpret_cast<const char *>(bytes.data());
379    result->SetValue(std::string(str, bytes.size()));
380    return result;
381  }
382  
383  // PythonString
384  
FromUTF8(llvm::StringRef string)385  Expected<PythonString> PythonString::FromUTF8(llvm::StringRef string) {
386    PyObject *str = PyUnicode_FromStringAndSize(string.data(), string.size());
387    if (!str)
388      return llvm::make_error<PythonException>();
389    return Take<PythonString>(str);
390  }
391  
PythonString(llvm::StringRef string)392  PythonString::PythonString(llvm::StringRef string) { SetString(string); }
393  
Check(PyObject * py_obj)394  bool PythonString::Check(PyObject *py_obj) {
395    if (!py_obj)
396      return false;
397  
398    if (PyUnicode_Check(py_obj))
399      return true;
400    return false;
401  }
402  
GetString() const403  llvm::StringRef PythonString::GetString() const {
404    auto s = AsUTF8();
405    if (!s) {
406      llvm::consumeError(s.takeError());
407      return llvm::StringRef("");
408    }
409    return s.get();
410  }
411  
AsUTF8() const412  Expected<llvm::StringRef> PythonString::AsUTF8() const {
413    if (!IsValid())
414      return nullDeref();
415  
416    Py_ssize_t size;
417    const char *data;
418  
419    data = PyUnicode_AsUTF8AndSize(m_py_obj, &size);
420  
421    if (!data)
422      return exception();
423  
424    return llvm::StringRef(data, size);
425  }
426  
GetSize() const427  size_t PythonString::GetSize() const {
428    if (IsValid()) {
429  #if PY_MINOR_VERSION >= 3
430      return PyUnicode_GetLength(m_py_obj);
431  #else
432      return PyUnicode_GetSize(m_py_obj);
433  #endif
434    }
435    return 0;
436  }
437  
SetString(llvm::StringRef string)438  void PythonString::SetString(llvm::StringRef string) {
439    auto s = FromUTF8(string);
440    if (!s) {
441      llvm::consumeError(s.takeError());
442      Reset();
443    } else {
444      *this = std::move(s.get());
445    }
446  }
447  
CreateStructuredString() const448  StructuredData::StringSP PythonString::CreateStructuredString() const {
449    StructuredData::StringSP result(new StructuredData::String);
450    result->SetValue(GetString());
451    return result;
452  }
453  
454  // PythonInteger
455  
PythonInteger(int64_t value)456  PythonInteger::PythonInteger(int64_t value) { SetInteger(value); }
457  
Check(PyObject * py_obj)458  bool PythonInteger::Check(PyObject *py_obj) {
459    if (!py_obj)
460      return false;
461  
462    // Python 3 does not have PyInt_Check.  There is only one type of integral
463    // value, long.
464    return PyLong_Check(py_obj);
465  }
466  
SetInteger(int64_t value)467  void PythonInteger::SetInteger(int64_t value) {
468    *this = Take<PythonInteger>(PyLong_FromLongLong(value));
469  }
470  
CreateStructuredInteger() const471  StructuredData::IntegerSP PythonInteger::CreateStructuredInteger() const {
472    StructuredData::UnsignedIntegerSP uint_sp = CreateStructuredUnsignedInteger();
473    return uint_sp ? StructuredData::IntegerSP(uint_sp)
474                   : CreateStructuredSignedInteger();
475  }
476  
477  StructuredData::UnsignedIntegerSP
CreateStructuredUnsignedInteger() const478  PythonInteger::CreateStructuredUnsignedInteger() const {
479    StructuredData::UnsignedIntegerSP result = nullptr;
480    llvm::Expected<unsigned long long> value = AsUnsignedLongLong();
481    if (!value)
482      llvm::consumeError(value.takeError());
483    else
484      result = std::make_shared<StructuredData::UnsignedInteger>(value.get());
485  
486    return result;
487  }
488  
489  StructuredData::SignedIntegerSP
CreateStructuredSignedInteger() const490  PythonInteger::CreateStructuredSignedInteger() const {
491    StructuredData::SignedIntegerSP result = nullptr;
492    llvm::Expected<long long> value = AsLongLong();
493    if (!value)
494      llvm::consumeError(value.takeError());
495    else
496      result = std::make_shared<StructuredData::SignedInteger>(value.get());
497  
498    return result;
499  }
500  
501  // PythonBoolean
502  
PythonBoolean(bool value)503  PythonBoolean::PythonBoolean(bool value) {
504    SetValue(value);
505  }
506  
Check(PyObject * py_obj)507  bool PythonBoolean::Check(PyObject *py_obj) {
508    return py_obj ? PyBool_Check(py_obj) : false;
509  }
510  
GetValue() const511  bool PythonBoolean::GetValue() const {
512    return m_py_obj ? PyObject_IsTrue(m_py_obj) : false;
513  }
514  
SetValue(bool value)515  void PythonBoolean::SetValue(bool value) {
516    *this = Take<PythonBoolean>(PyBool_FromLong(value));
517  }
518  
CreateStructuredBoolean() const519  StructuredData::BooleanSP PythonBoolean::CreateStructuredBoolean() const {
520    StructuredData::BooleanSP result(new StructuredData::Boolean);
521    result->SetValue(GetValue());
522    return result;
523  }
524  
525  // PythonList
526  
PythonList(PyInitialValue value)527  PythonList::PythonList(PyInitialValue value) {
528    if (value == PyInitialValue::Empty)
529      *this = Take<PythonList>(PyList_New(0));
530  }
531  
PythonList(int list_size)532  PythonList::PythonList(int list_size) {
533    *this = Take<PythonList>(PyList_New(list_size));
534  }
535  
Check(PyObject * py_obj)536  bool PythonList::Check(PyObject *py_obj) {
537    if (!py_obj)
538      return false;
539    return PyList_Check(py_obj);
540  }
541  
GetSize() const542  uint32_t PythonList::GetSize() const {
543    if (IsValid())
544      return PyList_GET_SIZE(m_py_obj);
545    return 0;
546  }
547  
GetItemAtIndex(uint32_t index) const548  PythonObject PythonList::GetItemAtIndex(uint32_t index) const {
549    if (IsValid())
550      return PythonObject(PyRefType::Borrowed, PyList_GetItem(m_py_obj, index));
551    return PythonObject();
552  }
553  
SetItemAtIndex(uint32_t index,const PythonObject & object)554  void PythonList::SetItemAtIndex(uint32_t index, const PythonObject &object) {
555    if (IsAllocated() && object.IsValid()) {
556      // PyList_SetItem is documented to "steal" a reference, so we need to
557      // convert it to an owned reference by incrementing it.
558      Py_INCREF(object.get());
559      PyList_SetItem(m_py_obj, index, object.get());
560    }
561  }
562  
AppendItem(const PythonObject & object)563  void PythonList::AppendItem(const PythonObject &object) {
564    if (IsAllocated() && object.IsValid()) {
565      // `PyList_Append` does *not* steal a reference, so do not call `Py_INCREF`
566      // here like we do with `PyList_SetItem`.
567      PyList_Append(m_py_obj, object.get());
568    }
569  }
570  
CreateStructuredArray() const571  StructuredData::ArraySP PythonList::CreateStructuredArray() const {
572    StructuredData::ArraySP result(new StructuredData::Array);
573    uint32_t count = GetSize();
574    for (uint32_t i = 0; i < count; ++i) {
575      PythonObject obj = GetItemAtIndex(i);
576      result->AddItem(obj.CreateStructuredObject());
577    }
578    return result;
579  }
580  
581  // PythonTuple
582  
PythonTuple(PyInitialValue value)583  PythonTuple::PythonTuple(PyInitialValue value) {
584    if (value == PyInitialValue::Empty)
585      *this = Take<PythonTuple>(PyTuple_New(0));
586  }
587  
PythonTuple(int tuple_size)588  PythonTuple::PythonTuple(int tuple_size) {
589    *this = Take<PythonTuple>(PyTuple_New(tuple_size));
590  }
591  
PythonTuple(std::initializer_list<PythonObject> objects)592  PythonTuple::PythonTuple(std::initializer_list<PythonObject> objects) {
593    m_py_obj = PyTuple_New(objects.size());
594  
595    uint32_t idx = 0;
596    for (auto object : objects) {
597      if (object.IsValid())
598        SetItemAtIndex(idx, object);
599      idx++;
600    }
601  }
602  
PythonTuple(std::initializer_list<PyObject * > objects)603  PythonTuple::PythonTuple(std::initializer_list<PyObject *> objects) {
604    m_py_obj = PyTuple_New(objects.size());
605  
606    uint32_t idx = 0;
607    for (auto py_object : objects) {
608      PythonObject object(PyRefType::Borrowed, py_object);
609      if (object.IsValid())
610        SetItemAtIndex(idx, object);
611      idx++;
612    }
613  }
614  
Check(PyObject * py_obj)615  bool PythonTuple::Check(PyObject *py_obj) {
616    if (!py_obj)
617      return false;
618    return PyTuple_Check(py_obj);
619  }
620  
GetSize() const621  uint32_t PythonTuple::GetSize() const {
622    if (IsValid())
623      return PyTuple_GET_SIZE(m_py_obj);
624    return 0;
625  }
626  
GetItemAtIndex(uint32_t index) const627  PythonObject PythonTuple::GetItemAtIndex(uint32_t index) const {
628    if (IsValid())
629      return PythonObject(PyRefType::Borrowed, PyTuple_GetItem(m_py_obj, index));
630    return PythonObject();
631  }
632  
SetItemAtIndex(uint32_t index,const PythonObject & object)633  void PythonTuple::SetItemAtIndex(uint32_t index, const PythonObject &object) {
634    if (IsAllocated() && object.IsValid()) {
635      // PyTuple_SetItem is documented to "steal" a reference, so we need to
636      // convert it to an owned reference by incrementing it.
637      Py_INCREF(object.get());
638      PyTuple_SetItem(m_py_obj, index, object.get());
639    }
640  }
641  
CreateStructuredArray() const642  StructuredData::ArraySP PythonTuple::CreateStructuredArray() const {
643    StructuredData::ArraySP result(new StructuredData::Array);
644    uint32_t count = GetSize();
645    for (uint32_t i = 0; i < count; ++i) {
646      PythonObject obj = GetItemAtIndex(i);
647      result->AddItem(obj.CreateStructuredObject());
648    }
649    return result;
650  }
651  
652  // PythonDictionary
653  
PythonDictionary(PyInitialValue value)654  PythonDictionary::PythonDictionary(PyInitialValue value) {
655    if (value == PyInitialValue::Empty)
656      *this = Take<PythonDictionary>(PyDict_New());
657  }
658  
Check(PyObject * py_obj)659  bool PythonDictionary::Check(PyObject *py_obj) {
660    if (!py_obj)
661      return false;
662  
663    return PyDict_Check(py_obj);
664  }
665  
HasKey(const llvm::Twine & key) const666  bool PythonDictionary::HasKey(const llvm::Twine &key) const {
667    if (!IsValid())
668      return false;
669  
670    PythonString key_object(key.isSingleStringRef() ? key.getSingleStringRef()
671                                                    : key.str());
672  
673    if (int res = PyDict_Contains(m_py_obj, key_object.get()) > 0)
674      return res;
675  
676    PyErr_Print();
677    return false;
678  }
679  
GetSize() const680  uint32_t PythonDictionary::GetSize() const {
681    if (IsValid())
682      return PyDict_Size(m_py_obj);
683    return 0;
684  }
685  
GetKeys() const686  PythonList PythonDictionary::GetKeys() const {
687    if (IsValid())
688      return PythonList(PyRefType::Owned, PyDict_Keys(m_py_obj));
689    return PythonList(PyInitialValue::Invalid);
690  }
691  
GetItemForKey(const PythonObject & key) const692  PythonObject PythonDictionary::GetItemForKey(const PythonObject &key) const {
693    auto item = GetItem(key);
694    if (!item) {
695      llvm::consumeError(item.takeError());
696      return PythonObject();
697    }
698    return std::move(item.get());
699  }
700  
701  Expected<PythonObject>
GetItem(const PythonObject & key) const702  PythonDictionary::GetItem(const PythonObject &key) const {
703    if (!IsValid())
704      return nullDeref();
705    PyObject *o = PyDict_GetItemWithError(m_py_obj, key.get());
706    if (PyErr_Occurred())
707      return exception();
708    if (!o)
709      return keyError();
710    return Retain<PythonObject>(o);
711  }
712  
GetItem(const Twine & key) const713  Expected<PythonObject> PythonDictionary::GetItem(const Twine &key) const {
714    if (!IsValid())
715      return nullDeref();
716    PyObject *o = PyDict_GetItemString(m_py_obj, NullTerminated(key));
717    if (PyErr_Occurred())
718      return exception();
719    if (!o)
720      return keyError();
721    return Retain<PythonObject>(o);
722  }
723  
SetItem(const PythonObject & key,const PythonObject & value) const724  Error PythonDictionary::SetItem(const PythonObject &key,
725                                  const PythonObject &value) const {
726    if (!IsValid() || !value.IsValid())
727      return nullDeref();
728    int r = PyDict_SetItem(m_py_obj, key.get(), value.get());
729    if (r < 0)
730      return exception();
731    return Error::success();
732  }
733  
SetItem(const Twine & key,const PythonObject & value) const734  Error PythonDictionary::SetItem(const Twine &key,
735                                  const PythonObject &value) const {
736    if (!IsValid() || !value.IsValid())
737      return nullDeref();
738    int r = PyDict_SetItemString(m_py_obj, NullTerminated(key), value.get());
739    if (r < 0)
740      return exception();
741    return Error::success();
742  }
743  
SetItemForKey(const PythonObject & key,const PythonObject & value)744  void PythonDictionary::SetItemForKey(const PythonObject &key,
745                                       const PythonObject &value) {
746    Error error = SetItem(key, value);
747    if (error)
748      llvm::consumeError(std::move(error));
749  }
750  
751  StructuredData::DictionarySP
CreateStructuredDictionary() const752  PythonDictionary::CreateStructuredDictionary() const {
753    StructuredData::DictionarySP result(new StructuredData::Dictionary);
754    PythonList keys(GetKeys());
755    uint32_t num_keys = keys.GetSize();
756    for (uint32_t i = 0; i < num_keys; ++i) {
757      PythonObject key = keys.GetItemAtIndex(i);
758      PythonObject value = GetItemForKey(key);
759      StructuredData::ObjectSP structured_value = value.CreateStructuredObject();
760      result->AddItem(key.Str().GetString(), structured_value);
761    }
762    return result;
763  }
764  
BuiltinsModule()765  PythonModule PythonModule::BuiltinsModule() { return AddModule("builtins"); }
766  
MainModule()767  PythonModule PythonModule::MainModule() { return AddModule("__main__"); }
768  
AddModule(llvm::StringRef module)769  PythonModule PythonModule::AddModule(llvm::StringRef module) {
770    std::string str = module.str();
771    return PythonModule(PyRefType::Borrowed, PyImport_AddModule(str.c_str()));
772  }
773  
Import(const Twine & name)774  Expected<PythonModule> PythonModule::Import(const Twine &name) {
775    PyObject *mod = PyImport_ImportModule(NullTerminated(name));
776    if (!mod)
777      return exception();
778    return Take<PythonModule>(mod);
779  }
780  
Get(const Twine & name)781  Expected<PythonObject> PythonModule::Get(const Twine &name) {
782    if (!IsValid())
783      return nullDeref();
784    PyObject *dict = PyModule_GetDict(m_py_obj);
785    if (!dict)
786      return exception();
787    PyObject *item = PyDict_GetItemString(dict, NullTerminated(name));
788    if (!item)
789      return exception();
790    return Retain<PythonObject>(item);
791  }
792  
Check(PyObject * py_obj)793  bool PythonModule::Check(PyObject *py_obj) {
794    if (!py_obj)
795      return false;
796  
797    return PyModule_Check(py_obj);
798  }
799  
GetDictionary() const800  PythonDictionary PythonModule::GetDictionary() const {
801    if (!IsValid())
802      return PythonDictionary();
803    return Retain<PythonDictionary>(PyModule_GetDict(m_py_obj));
804  }
805  
Check(PyObject * py_obj)806  bool PythonCallable::Check(PyObject *py_obj) {
807    if (!py_obj)
808      return false;
809  
810    return PyCallable_Check(py_obj);
811  }
812  
813  #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3
814  static const char get_arg_info_script[] = R"(
815  from inspect import signature, Parameter, ismethod
816  from collections import namedtuple
817  ArgInfo = namedtuple('ArgInfo', ['count', 'has_varargs'])
818  def main(f):
819      count = 0
820      varargs = False
821      for parameter in signature(f).parameters.values():
822          kind = parameter.kind
823          if kind in (Parameter.POSITIONAL_ONLY,
824                      Parameter.POSITIONAL_OR_KEYWORD):
825              count += 1
826          elif kind == Parameter.VAR_POSITIONAL:
827              varargs = True
828          elif kind in (Parameter.KEYWORD_ONLY,
829                        Parameter.VAR_KEYWORD):
830              pass
831          else:
832              raise Exception(f'unknown parameter kind: {kind}')
833      return ArgInfo(count, varargs)
834  )";
835  #endif
836  
GetArgInfo() const837  Expected<PythonCallable::ArgInfo> PythonCallable::GetArgInfo() const {
838    ArgInfo result = {};
839    if (!IsValid())
840      return nullDeref();
841  
842  #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3
843  
844    // no need to synchronize access to this global, we already have the GIL
845    static PythonScript get_arg_info(get_arg_info_script);
846    Expected<PythonObject> pyarginfo = get_arg_info(*this);
847    if (!pyarginfo)
848      return pyarginfo.takeError();
849    long long count =
850        cantFail(As<long long>(pyarginfo.get().GetAttribute("count")));
851    bool has_varargs =
852        cantFail(As<bool>(pyarginfo.get().GetAttribute("has_varargs")));
853    result.max_positional_args = has_varargs ? ArgInfo::UNBOUNDED : count;
854  
855  #else
856    PyObject *py_func_obj;
857    bool is_bound_method = false;
858    bool is_class = false;
859  
860    if (PyType_Check(m_py_obj) || PyClass_Check(m_py_obj)) {
861      auto init = GetAttribute("__init__");
862      if (!init)
863        return init.takeError();
864      py_func_obj = init.get().get();
865      is_class = true;
866    } else {
867      py_func_obj = m_py_obj;
868    }
869  
870    if (PyMethod_Check(py_func_obj)) {
871      py_func_obj = PyMethod_GET_FUNCTION(py_func_obj);
872      PythonObject im_self = GetAttributeValue("im_self");
873      if (im_self.IsValid() && !im_self.IsNone())
874        is_bound_method = true;
875    } else {
876      // see if this is a callable object with an __call__ method
877      if (!PyFunction_Check(py_func_obj)) {
878        PythonObject __call__ = GetAttributeValue("__call__");
879        if (__call__.IsValid()) {
880          auto __callable__ = __call__.AsType<PythonCallable>();
881          if (__callable__.IsValid()) {
882            py_func_obj = PyMethod_GET_FUNCTION(__callable__.get());
883            PythonObject im_self = __callable__.GetAttributeValue("im_self");
884            if (im_self.IsValid() && !im_self.IsNone())
885              is_bound_method = true;
886          }
887        }
888      }
889    }
890  
891    if (!py_func_obj)
892      return result;
893  
894    PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(py_func_obj);
895    if (!code)
896      return result;
897  
898    auto count = code->co_argcount;
899    bool has_varargs = !!(code->co_flags & CO_VARARGS);
900    result.max_positional_args =
901        has_varargs ? ArgInfo::UNBOUNDED
902                    : (count - (int)is_bound_method) - (int)is_class;
903  
904  #endif
905  
906    return result;
907  }
908  
909  constexpr unsigned
910      PythonCallable::ArgInfo::UNBOUNDED; // FIXME delete after c++17
911  
operator ()()912  PythonObject PythonCallable::operator()() {
913    return PythonObject(PyRefType::Owned, PyObject_CallObject(m_py_obj, nullptr));
914  }
915  
916  PythonObject PythonCallable::
operator ()(std::initializer_list<PyObject * > args)917  operator()(std::initializer_list<PyObject *> args) {
918    PythonTuple arg_tuple(args);
919    return PythonObject(PyRefType::Owned,
920                        PyObject_CallObject(m_py_obj, arg_tuple.get()));
921  }
922  
923  PythonObject PythonCallable::
operator ()(std::initializer_list<PythonObject> args)924  operator()(std::initializer_list<PythonObject> args) {
925    PythonTuple arg_tuple(args);
926    return PythonObject(PyRefType::Owned,
927                        PyObject_CallObject(m_py_obj, arg_tuple.get()));
928  }
929  
Check(PyObject * py_obj)930  bool PythonFile::Check(PyObject *py_obj) {
931    if (!py_obj)
932      return false;
933    // In Python 3, there is no `PyFile_Check`, and in fact PyFile is not even a
934    // first-class object type anymore.  `PyFile_FromFd` is just a thin wrapper
935    // over `io.open()`, which returns some object derived from `io.IOBase`. As a
936    // result, the only way to detect a file in Python 3 is to check whether it
937    // inherits from `io.IOBase`.
938    auto io_module = PythonModule::Import("io");
939    if (!io_module) {
940      llvm::consumeError(io_module.takeError());
941      return false;
942    }
943    auto iobase = io_module.get().Get("IOBase");
944    if (!iobase) {
945      llvm::consumeError(iobase.takeError());
946      return false;
947    }
948    int r = PyObject_IsInstance(py_obj, iobase.get().get());
949    if (r < 0) {
950      llvm::consumeError(exception()); // clear the exception and log it.
951      return false;
952    }
953    return !!r;
954  }
955  
toCString() const956  const char *PythonException::toCString() const {
957    if (!m_repr_bytes)
958      return "unknown exception";
959    return PyBytes_AS_STRING(m_repr_bytes);
960  }
961  
PythonException(const char * caller)962  PythonException::PythonException(const char *caller) {
963    assert(PyErr_Occurred());
964    m_exception_type = m_exception = m_traceback = m_repr_bytes = nullptr;
965    PyErr_Fetch(&m_exception_type, &m_exception, &m_traceback);
966    PyErr_NormalizeException(&m_exception_type, &m_exception, &m_traceback);
967    PyErr_Clear();
968    if (m_exception) {
969      PyObject *repr = PyObject_Repr(m_exception);
970      if (repr) {
971        m_repr_bytes = PyUnicode_AsEncodedString(repr, "utf-8", nullptr);
972        if (!m_repr_bytes) {
973          PyErr_Clear();
974        }
975        Py_XDECREF(repr);
976      } else {
977        PyErr_Clear();
978      }
979    }
980    Log *log = GetLog(LLDBLog::Script);
981    if (caller)
982      LLDB_LOGF(log, "%s failed with exception: %s", caller, toCString());
983    else
984      LLDB_LOGF(log, "python exception: %s", toCString());
985  }
Restore()986  void PythonException::Restore() {
987    if (m_exception_type && m_exception) {
988      PyErr_Restore(m_exception_type, m_exception, m_traceback);
989    } else {
990      PyErr_SetString(PyExc_Exception, toCString());
991    }
992    m_exception_type = m_exception = m_traceback = nullptr;
993  }
994  
~PythonException()995  PythonException::~PythonException() {
996    Py_XDECREF(m_exception_type);
997    Py_XDECREF(m_exception);
998    Py_XDECREF(m_traceback);
999    Py_XDECREF(m_repr_bytes);
1000  }
1001  
log(llvm::raw_ostream & OS) const1002  void PythonException::log(llvm::raw_ostream &OS) const { OS << toCString(); }
1003  
convertToErrorCode() const1004  std::error_code PythonException::convertToErrorCode() const {
1005    return llvm::inconvertibleErrorCode();
1006  }
1007  
Matches(PyObject * exc) const1008  bool PythonException::Matches(PyObject *exc) const {
1009    return PyErr_GivenExceptionMatches(m_exception_type, exc);
1010  }
1011  
1012  const char read_exception_script[] = R"(
1013  import sys
1014  from traceback import print_exception
1015  if sys.version_info.major < 3:
1016    from StringIO import StringIO
1017  else:
1018    from io import StringIO
1019  def main(exc_type, exc_value, tb):
1020    f = StringIO()
1021    print_exception(exc_type, exc_value, tb, file=f)
1022    return f.getvalue()
1023  )";
1024  
ReadBacktrace() const1025  std::string PythonException::ReadBacktrace() const {
1026  
1027    if (!m_traceback)
1028      return toCString();
1029  
1030    // no need to synchronize access to this global, we already have the GIL
1031    static PythonScript read_exception(read_exception_script);
1032  
1033    Expected<std::string> backtrace = As<std::string>(
1034        read_exception(m_exception_type, m_exception, m_traceback));
1035  
1036    if (!backtrace) {
1037      std::string message =
1038          std::string(toCString()) + "\n" +
1039          "Traceback unavailable, an error occurred while reading it:\n";
1040      return (message + llvm::toString(backtrace.takeError()));
1041    }
1042  
1043    return std::move(backtrace.get());
1044  }
1045  
1046  char PythonException::ID = 0;
1047  
1048  llvm::Expected<File::OpenOptions>
GetOptionsForPyObject(const PythonObject & obj)1049  GetOptionsForPyObject(const PythonObject &obj) {
1050    auto options = File::OpenOptions(0);
1051    auto readable = As<bool>(obj.CallMethod("readable"));
1052    if (!readable)
1053      return readable.takeError();
1054    auto writable = As<bool>(obj.CallMethod("writable"));
1055    if (!writable)
1056      return writable.takeError();
1057    if (readable.get() && writable.get())
1058      options |= File::eOpenOptionReadWrite;
1059    else if (writable.get())
1060      options |= File::eOpenOptionWriteOnly;
1061    else if (readable.get())
1062      options |= File::eOpenOptionReadOnly;
1063    return options;
1064  }
1065  
1066  // Base class template for python files.   All it knows how to do
1067  // is hold a reference to the python object and close or flush it
1068  // when the File is closed.
1069  namespace {
1070  template <typename Base> class OwnedPythonFile : public Base {
1071  public:
1072    template <typename... Args>
OwnedPythonFile(const PythonFile & file,bool borrowed,Args...args)1073    OwnedPythonFile(const PythonFile &file, bool borrowed, Args... args)
1074        : Base(args...), m_py_obj(file), m_borrowed(borrowed) {
1075      assert(m_py_obj);
1076    }
1077  
~OwnedPythonFile()1078    ~OwnedPythonFile() override {
1079      assert(m_py_obj);
1080      GIL takeGIL;
1081      Close();
1082      // we need to ensure the python object is released while we still
1083      // hold the GIL
1084      m_py_obj.Reset();
1085    }
1086  
IsPythonSideValid() const1087    bool IsPythonSideValid() const {
1088      GIL takeGIL;
1089      auto closed = As<bool>(m_py_obj.GetAttribute("closed"));
1090      if (!closed) {
1091        llvm::consumeError(closed.takeError());
1092        return false;
1093      }
1094      return !closed.get();
1095    }
1096  
IsValid() const1097    bool IsValid() const override {
1098      return IsPythonSideValid() && Base::IsValid();
1099    }
1100  
Close()1101    Status Close() override {
1102      assert(m_py_obj);
1103      Status py_error, base_error;
1104      GIL takeGIL;
1105      if (!m_borrowed) {
1106        auto r = m_py_obj.CallMethod("close");
1107        if (!r)
1108          py_error = Status(r.takeError());
1109      }
1110      base_error = Base::Close();
1111      if (py_error.Fail())
1112        return py_error;
1113      return base_error;
1114    };
1115  
GetPythonObject() const1116    PyObject *GetPythonObject() const {
1117      assert(m_py_obj.IsValid());
1118      return m_py_obj.get();
1119    }
1120  
1121    static bool classof(const File *file) = delete;
1122  
1123  protected:
1124    PythonFile m_py_obj;
1125    bool m_borrowed;
1126  };
1127  } // namespace
1128  
1129  // A SimplePythonFile is a OwnedPythonFile that just does all I/O as
1130  // a NativeFile
1131  namespace {
1132  class SimplePythonFile : public OwnedPythonFile<NativeFile> {
1133  public:
SimplePythonFile(const PythonFile & file,bool borrowed,int fd,File::OpenOptions options)1134    SimplePythonFile(const PythonFile &file, bool borrowed, int fd,
1135                     File::OpenOptions options)
1136        : OwnedPythonFile(file, borrowed, fd, options, false) {}
1137  
1138    static char ID;
isA(const void * classID) const1139    bool isA(const void *classID) const override {
1140      return classID == &ID || NativeFile::isA(classID);
1141    }
classof(const File * file)1142    static bool classof(const File *file) { return file->isA(&ID); }
1143  };
1144  char SimplePythonFile::ID = 0;
1145  } // namespace
1146  
1147  namespace {
1148  class PythonBuffer {
1149  public:
1150    PythonBuffer &operator=(const PythonBuffer &) = delete;
1151    PythonBuffer(const PythonBuffer &) = delete;
1152  
Create(PythonObject & obj,int flags=PyBUF_SIMPLE)1153    static Expected<PythonBuffer> Create(PythonObject &obj,
1154                                         int flags = PyBUF_SIMPLE) {
1155      Py_buffer py_buffer = {};
1156      PyObject_GetBuffer(obj.get(), &py_buffer, flags);
1157      if (!py_buffer.obj)
1158        return llvm::make_error<PythonException>();
1159      return PythonBuffer(py_buffer);
1160    }
1161  
PythonBuffer(PythonBuffer && other)1162    PythonBuffer(PythonBuffer &&other) {
1163      m_buffer = other.m_buffer;
1164      other.m_buffer.obj = nullptr;
1165    }
1166  
~PythonBuffer()1167    ~PythonBuffer() {
1168      if (m_buffer.obj)
1169        PyBuffer_Release(&m_buffer);
1170    }
1171  
get()1172    Py_buffer &get() { return m_buffer; }
1173  
1174  private:
1175    // takes ownership of the buffer.
PythonBuffer(const Py_buffer & py_buffer)1176    PythonBuffer(const Py_buffer &py_buffer) : m_buffer(py_buffer) {}
1177    Py_buffer m_buffer;
1178  };
1179  } // namespace
1180  
1181  // Shared methods between TextPythonFile and BinaryPythonFile
1182  namespace {
1183  class PythonIOFile : public OwnedPythonFile<File> {
1184  public:
PythonIOFile(const PythonFile & file,bool borrowed)1185    PythonIOFile(const PythonFile &file, bool borrowed)
1186        : OwnedPythonFile(file, borrowed) {}
1187  
~PythonIOFile()1188    ~PythonIOFile() override { Close(); }
1189  
IsValid() const1190    bool IsValid() const override { return IsPythonSideValid(); }
1191  
Close()1192    Status Close() override {
1193      assert(m_py_obj);
1194      GIL takeGIL;
1195      if (m_borrowed)
1196        return Flush();
1197      auto r = m_py_obj.CallMethod("close");
1198      if (!r)
1199        return Status(r.takeError());
1200      return Status();
1201    }
1202  
Flush()1203    Status Flush() override {
1204      GIL takeGIL;
1205      auto r = m_py_obj.CallMethod("flush");
1206      if (!r)
1207        return Status(r.takeError());
1208      return Status();
1209    }
1210  
GetOptions() const1211    Expected<File::OpenOptions> GetOptions() const override {
1212      GIL takeGIL;
1213      return GetOptionsForPyObject(m_py_obj);
1214    }
1215  
1216    static char ID;
isA(const void * classID) const1217    bool isA(const void *classID) const override {
1218      return classID == &ID || File::isA(classID);
1219    }
classof(const File * file)1220    static bool classof(const File *file) { return file->isA(&ID); }
1221  };
1222  char PythonIOFile::ID = 0;
1223  } // namespace
1224  
1225  namespace {
1226  class BinaryPythonFile : public PythonIOFile {
1227  protected:
1228    int m_descriptor;
1229  
1230  public:
BinaryPythonFile(int fd,const PythonFile & file,bool borrowed)1231    BinaryPythonFile(int fd, const PythonFile &file, bool borrowed)
1232        : PythonIOFile(file, borrowed),
1233          m_descriptor(File::DescriptorIsValid(fd) ? fd
1234                                                   : File::kInvalidDescriptor) {}
1235  
GetDescriptor() const1236    int GetDescriptor() const override { return m_descriptor; }
1237  
Write(const void * buf,size_t & num_bytes)1238    Status Write(const void *buf, size_t &num_bytes) override {
1239      GIL takeGIL;
1240      PyObject *pybuffer_p = PyMemoryView_FromMemory(
1241          const_cast<char *>((const char *)buf), num_bytes, PyBUF_READ);
1242      if (!pybuffer_p)
1243        return Status(llvm::make_error<PythonException>());
1244      auto pybuffer = Take<PythonObject>(pybuffer_p);
1245      num_bytes = 0;
1246      auto bytes_written = As<long long>(m_py_obj.CallMethod("write", pybuffer));
1247      if (!bytes_written)
1248        return Status(bytes_written.takeError());
1249      if (bytes_written.get() < 0)
1250        return Status(".write() method returned a negative number!");
1251      static_assert(sizeof(long long) >= sizeof(size_t), "overflow");
1252      num_bytes = bytes_written.get();
1253      return Status();
1254    }
1255  
Read(void * buf,size_t & num_bytes)1256    Status Read(void *buf, size_t &num_bytes) override {
1257      GIL takeGIL;
1258      static_assert(sizeof(long long) >= sizeof(size_t), "overflow");
1259      auto pybuffer_obj =
1260          m_py_obj.CallMethod("read", (unsigned long long)num_bytes);
1261      if (!pybuffer_obj)
1262        return Status(pybuffer_obj.takeError());
1263      num_bytes = 0;
1264      if (pybuffer_obj.get().IsNone()) {
1265        // EOF
1266        num_bytes = 0;
1267        return Status();
1268      }
1269      auto pybuffer = PythonBuffer::Create(pybuffer_obj.get());
1270      if (!pybuffer)
1271        return Status(pybuffer.takeError());
1272      memcpy(buf, pybuffer.get().get().buf, pybuffer.get().get().len);
1273      num_bytes = pybuffer.get().get().len;
1274      return Status();
1275    }
1276  };
1277  } // namespace
1278  
1279  namespace {
1280  class TextPythonFile : public PythonIOFile {
1281  protected:
1282    int m_descriptor;
1283  
1284  public:
TextPythonFile(int fd,const PythonFile & file,bool borrowed)1285    TextPythonFile(int fd, const PythonFile &file, bool borrowed)
1286        : PythonIOFile(file, borrowed),
1287          m_descriptor(File::DescriptorIsValid(fd) ? fd
1288                                                   : File::kInvalidDescriptor) {}
1289  
GetDescriptor() const1290    int GetDescriptor() const override { return m_descriptor; }
1291  
Write(const void * buf,size_t & num_bytes)1292    Status Write(const void *buf, size_t &num_bytes) override {
1293      GIL takeGIL;
1294      auto pystring =
1295          PythonString::FromUTF8(llvm::StringRef((const char *)buf, num_bytes));
1296      if (!pystring)
1297        return Status(pystring.takeError());
1298      num_bytes = 0;
1299      auto bytes_written =
1300          As<long long>(m_py_obj.CallMethod("write", pystring.get()));
1301      if (!bytes_written)
1302        return Status(bytes_written.takeError());
1303      if (bytes_written.get() < 0)
1304        return Status(".write() method returned a negative number!");
1305      static_assert(sizeof(long long) >= sizeof(size_t), "overflow");
1306      num_bytes = bytes_written.get();
1307      return Status();
1308    }
1309  
Read(void * buf,size_t & num_bytes)1310    Status Read(void *buf, size_t &num_bytes) override {
1311      GIL takeGIL;
1312      size_t num_chars = num_bytes / 6;
1313      size_t orig_num_bytes = num_bytes;
1314      num_bytes = 0;
1315      if (orig_num_bytes < 6) {
1316        return Status("can't read less than 6 bytes from a utf8 text stream");
1317      }
1318      auto pystring = As<PythonString>(
1319          m_py_obj.CallMethod("read", (unsigned long long)num_chars));
1320      if (!pystring)
1321        return Status(pystring.takeError());
1322      if (pystring.get().IsNone()) {
1323        // EOF
1324        return Status();
1325      }
1326      auto stringref = pystring.get().AsUTF8();
1327      if (!stringref)
1328        return Status(stringref.takeError());
1329      num_bytes = stringref.get().size();
1330      memcpy(buf, stringref.get().begin(), num_bytes);
1331      return Status();
1332    }
1333  };
1334  } // namespace
1335  
ConvertToFile(bool borrowed)1336  llvm::Expected<FileSP> PythonFile::ConvertToFile(bool borrowed) {
1337    if (!IsValid())
1338      return llvm::createStringError(llvm::inconvertibleErrorCode(),
1339                                     "invalid PythonFile");
1340  
1341    int fd = PyObject_AsFileDescriptor(m_py_obj);
1342    if (fd < 0) {
1343      PyErr_Clear();
1344      return ConvertToFileForcingUseOfScriptingIOMethods(borrowed);
1345    }
1346    auto options = GetOptionsForPyObject(*this);
1347    if (!options)
1348      return options.takeError();
1349  
1350    File::OpenOptions rw =
1351        options.get() & (File::eOpenOptionReadOnly | File::eOpenOptionWriteOnly |
1352                         File::eOpenOptionReadWrite);
1353    if (rw == File::eOpenOptionWriteOnly || rw == File::eOpenOptionReadWrite) {
1354      // LLDB and python will not share I/O buffers.  We should probably
1355      // flush the python buffers now.
1356      auto r = CallMethod("flush");
1357      if (!r)
1358        return r.takeError();
1359    }
1360  
1361    FileSP file_sp;
1362    if (borrowed) {
1363      // In this case we don't need to retain the python
1364      // object at all.
1365      file_sp = std::make_shared<NativeFile>(fd, options.get(), false);
1366    } else {
1367      file_sp = std::static_pointer_cast<File>(
1368          std::make_shared<SimplePythonFile>(*this, borrowed, fd, options.get()));
1369    }
1370    if (!file_sp->IsValid())
1371      return llvm::createStringError(llvm::inconvertibleErrorCode(),
1372                                     "invalid File");
1373  
1374    return file_sp;
1375  }
1376  
1377  llvm::Expected<FileSP>
ConvertToFileForcingUseOfScriptingIOMethods(bool borrowed)1378  PythonFile::ConvertToFileForcingUseOfScriptingIOMethods(bool borrowed) {
1379  
1380    assert(!PyErr_Occurred());
1381  
1382    if (!IsValid())
1383      return llvm::createStringError(llvm::inconvertibleErrorCode(),
1384                                     "invalid PythonFile");
1385  
1386    int fd = PyObject_AsFileDescriptor(m_py_obj);
1387    if (fd < 0) {
1388      PyErr_Clear();
1389      fd = File::kInvalidDescriptor;
1390    }
1391  
1392    auto io_module = PythonModule::Import("io");
1393    if (!io_module)
1394      return io_module.takeError();
1395    auto textIOBase = io_module.get().Get("TextIOBase");
1396    if (!textIOBase)
1397      return textIOBase.takeError();
1398    auto rawIOBase = io_module.get().Get("RawIOBase");
1399    if (!rawIOBase)
1400      return rawIOBase.takeError();
1401    auto bufferedIOBase = io_module.get().Get("BufferedIOBase");
1402    if (!bufferedIOBase)
1403      return bufferedIOBase.takeError();
1404  
1405    FileSP file_sp;
1406  
1407    auto isTextIO = IsInstance(textIOBase.get());
1408    if (!isTextIO)
1409      return isTextIO.takeError();
1410    if (isTextIO.get())
1411      file_sp = std::static_pointer_cast<File>(
1412          std::make_shared<TextPythonFile>(fd, *this, borrowed));
1413  
1414    auto isRawIO = IsInstance(rawIOBase.get());
1415    if (!isRawIO)
1416      return isRawIO.takeError();
1417    auto isBufferedIO = IsInstance(bufferedIOBase.get());
1418    if (!isBufferedIO)
1419      return isBufferedIO.takeError();
1420  
1421    if (isRawIO.get() || isBufferedIO.get()) {
1422      file_sp = std::static_pointer_cast<File>(
1423          std::make_shared<BinaryPythonFile>(fd, *this, borrowed));
1424    }
1425  
1426    if (!file_sp)
1427      return llvm::createStringError(llvm::inconvertibleErrorCode(),
1428                                     "python file is neither text nor binary");
1429  
1430    if (!file_sp->IsValid())
1431      return llvm::createStringError(llvm::inconvertibleErrorCode(),
1432                                     "invalid File");
1433  
1434    return file_sp;
1435  }
1436  
FromFile(File & file,const char * mode)1437  Expected<PythonFile> PythonFile::FromFile(File &file, const char *mode) {
1438    if (!file.IsValid())
1439      return llvm::createStringError(llvm::inconvertibleErrorCode(),
1440                                     "invalid file");
1441  
1442    if (auto *simple = llvm::dyn_cast<SimplePythonFile>(&file))
1443      return Retain<PythonFile>(simple->GetPythonObject());
1444    if (auto *pythonio = llvm::dyn_cast<PythonIOFile>(&file))
1445      return Retain<PythonFile>(pythonio->GetPythonObject());
1446  
1447    if (!mode) {
1448      auto m = file.GetOpenMode();
1449      if (!m)
1450        return m.takeError();
1451      mode = m.get();
1452    }
1453  
1454    PyObject *file_obj;
1455    file_obj = PyFile_FromFd(file.GetDescriptor(), nullptr, mode, -1, nullptr,
1456                             "ignore", nullptr, /*closefd=*/0);
1457  
1458    if (!file_obj)
1459      return exception();
1460  
1461    return Take<PythonFile>(file_obj);
1462  }
1463  
Init()1464  Error PythonScript::Init() {
1465    if (function.IsValid())
1466      return Error::success();
1467  
1468    PythonDictionary globals(PyInitialValue::Empty);
1469    auto builtins = PythonModule::BuiltinsModule();
1470    if (Error error = globals.SetItem("__builtins__", builtins))
1471      return error;
1472    PyObject *o =
1473        PyRun_String(script, Py_file_input, globals.get(), globals.get());
1474    if (!o)
1475      return exception();
1476    Take<PythonObject>(o);
1477    auto f = As<PythonCallable>(globals.GetItem("main"));
1478    if (!f)
1479      return f.takeError();
1480    function = std::move(f.get());
1481  
1482    return Error::success();
1483  }
1484  
1485  llvm::Expected<PythonObject>
runStringOneLine(const llvm::Twine & string,const PythonDictionary & globals,const PythonDictionary & locals)1486  python::runStringOneLine(const llvm::Twine &string,
1487                           const PythonDictionary &globals,
1488                           const PythonDictionary &locals) {
1489    if (!globals.IsValid() || !locals.IsValid())
1490      return nullDeref();
1491  
1492    PyObject *code =
1493        Py_CompileString(NullTerminated(string), "<string>", Py_eval_input);
1494    if (!code) {
1495      PyErr_Clear();
1496      code =
1497          Py_CompileString(NullTerminated(string), "<string>", Py_single_input);
1498    }
1499    if (!code)
1500      return exception();
1501    auto code_ref = Take<PythonObject>(code);
1502  
1503    PyObject *result = PyEval_EvalCode(code, globals.get(), locals.get());
1504  
1505    if (!result)
1506      return exception();
1507  
1508    return Take<PythonObject>(result);
1509  }
1510  
1511  llvm::Expected<PythonObject>
runStringMultiLine(const llvm::Twine & string,const PythonDictionary & globals,const PythonDictionary & locals)1512  python::runStringMultiLine(const llvm::Twine &string,
1513                             const PythonDictionary &globals,
1514                             const PythonDictionary &locals) {
1515    if (!globals.IsValid() || !locals.IsValid())
1516      return nullDeref();
1517    PyObject *result = PyRun_String(NullTerminated(string), Py_file_input,
1518                                    globals.get(), locals.get());
1519    if (!result)
1520      return exception();
1521    return Take<PythonObject>(result);
1522  }
1523  
1524  #endif
1525